mirror of
https://github.com/RYDE-WORK/lnp_ml.git
synced 2026-03-21 09:36:32 +08:00
53 lines
1.6 KiB
Python
53 lines
1.6 KiB
Python
"""数据清洗脚本：修正原始数据中的问题"""
|
|
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import typer
|
|
from loguru import logger
|
|
|
|
from lnp_ml.config import RAW_DATA_DIR, INTERIM_DATA_DIR
|
|
|
|
|
|
app = typer.Typer()
|
|
|
|
|
|
@app.command()
|
|
def main(
|
|
input_path: Path = RAW_DATA_DIR / "internal.xlsx",
|
|
output_path: Path = INTERIM_DATA_DIR / "internal.csv",
|
|
):
|
|
"""
|
|
预处理内部数据，按给药途径分组进行 z-score 标准化，对 size 列取 log。
|
|
|
|
修正内容：
|
|
1. 按给药途径分组进行 z-score 标准化
|
|
2. 对 size 列取 log
|
|
"""
|
|
logger.info(f"Loading data from {input_path}")
|
|
df = pd.read_excel(input_path, header=2)
|
|
logger.info(f"Loaded {len(df)} samples")
|
|
|
|
# 分别对肌肉注射组和静脉注射组重新进行 z-score 标准化
|
|
logger.info("Z-score normalizing delivery by Route_of_administration...")
|
|
df["unnormalized_delivery"] = pd.to_numeric(df["unnormalized_delivery"], errors="coerce")
|
|
df["quantified_delivery"] = (
|
|
df.groupby("Route_of_administration")["unnormalized_delivery"]
|
|
.transform(lambda x: (x - x.mean()) / x.std())
|
|
)
|
|
|
|
# 对 size 列取 log
|
|
logger.info("Log-transforming size column...")
|
|
df["size"] = pd.to_numeric(df["size"], errors="coerce")
|
|
df["size"] = np.log(df["size"].replace(0, np.nan)) # 避免 log(0)
|
|
|
|
# 保存
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
df.to_csv(output_path, index=False)
|
|
logger.success(f"Saved cleaned data to {output_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app()
|