"""数据清洗脚本:修正原始数据中的问题""" from pathlib import Path import numpy as np import pandas as pd import typer from loguru import logger from lnp_ml.config import RAW_DATA_DIR, INTERIM_DATA_DIR app = typer.Typer() @app.command() def main( input_path: Path = RAW_DATA_DIR / "internal.xlsx", output_path: Path = INTERIM_DATA_DIR / "internal.csv", ): """ 预处理内部数据,按给药途径分组进行 z-score 标准化,对 size 列取 log。 修正内容: 1. 按给药途径分组进行 z-score 标准化 2. 对 size 列取 log """ logger.info(f"Loading data from {input_path}") df = pd.read_excel(input_path, header=2) logger.info(f"Loaded {len(df)} samples") # 分别对肌肉注射组和静脉注射组重新进行 z-score 标准化 logger.info("Z-score normalizing delivery by Route_of_administration...") df["unnormalized_delivery"] = pd.to_numeric(df["unnormalized_delivery"], errors="coerce") df["quantified_delivery"] = ( df.groupby("Route_of_administration")["unnormalized_delivery"] .transform(lambda x: (x - x.mean()) / x.std()) ) # 对 size 列取 log logger.info("Log-transforming size column...") df["size"] = pd.to_numeric(df["size"], errors="coerce") df["size"] = np.log(df["size"].replace(0, np.nan)) # 避免 log(0) # 保存 output_path.parent.mkdir(parents=True, exist_ok=True) df.to_csv(output_path, index=False) logger.success(f"Saved cleaned data to {output_path}") if __name__ == "__main__": app()