drug-disease-mapping/mergeDrugs.py at main · MuhammadMuneeb007/drug-disease-mapping · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
#!/usr/bin/env python3

import polars as pl
from pathlib import Path

# =========================
# Helper: normalize schema
# =========================
def normalize_df(
    df: pl.DataFrame,
    *,
    drug_name_col: str,
    disease_col: str,
    source: str,
    internal_source: str,
    drug_id_col: str | None = None,
) -> pl.DataFrame:

    return df.select([
        pl.col(drug_name_col).cast(pl.Utf8).alias("drug_name"),

        (
            pl.col(drug_id_col).cast(pl.Utf8)
            if drug_id_col and drug_id_col in df.columns
            else pl.lit(None).cast(pl.Utf8)
        ).alias("drug_id"),

        pl.col(disease_col).cast(pl.Utf8).alias("disease"),
        pl.lit(source).alias("source"),
        pl.lit(internal_source).alias("internal_source"),
    ])


# =========================
# Load files
# =========================
base = Path(".")

aact        = pl.read_csv(base / "aact_drug_disease.csv")
ctgov       = pl.read_csv(base / "ctgov_drug_disease.csv")
chembl      = pl.read_csv(base / "chembl.csv")
ctd         = pl.read_csv(base / "ctd_drug_disease.csv")
drugcentral = pl.read_csv(base / "drugcentral_drug_disease.csv")
opentargets = pl.read_csv(base / "opentargets_drug_disease.csv")
sider       = pl.read_csv(base / "sider_drug_indication.csv")

# =========================
# Normalize all sources
# =========================
dfs = [

    normalize_df(
        aact,
        drug_name_col="drug_name",
        disease_col="disease_name",
        source="AACT",
        internal_source="ClinicalTrials.gov",
    ),

    normalize_df(
        ctgov,
        drug_name_col="drug_name",
        disease_col="disease_name",
        source="CTGOV",
        internal_source="ClinicalTrials.gov",
    ),

    normalize_df(
        chembl,
        drug_name_col="drug_name",
        drug_id_col="drug_id",
        disease_col="disease_name",
        source="ChEMBL",
        internal_source="ChEMBL",
    ),

    normalize_df(
        ctd,
        drug_name_col="drug_name",
        drug_id_col="drug_id",
        disease_col="disease_name",
        source="CTD",
        internal_source="CTD_aggregate",
    ),

    normalize_df(
        drugcentral,
        drug_name_col="drug_name",
        drug_id_col="drug_id",
        disease_col="disease_name",
        source="DrugCentral",
        internal_source="DrugCentral",
    ),

    normalize_df(
        opentargets,
        drug_name_col="drug_name",
        drug_id_col="drug_id",
        disease_col="disease_name",
        source="OpenTargets",
        internal_source="OpenTargets",
    ),

    normalize_df(
        sider,
        drug_name_col="drug_name",
        drug_id_col="drug_id",
        disease_col="indication_name",
        source="SIDER",
        internal_source="SIDER4.1",
    ),
]

# =========================
# Merge + normalize text
# =========================
merged = pl.concat(dfs, how="vertical")

merged = merged.with_columns([
    pl.col("drug_name").str.to_lowercase().str.strip_chars(),
    pl.col("disease").str.to_lowercase().str.strip_chars(),
    pl.col("drug_id").str.strip_chars(),
])

# =========================
# DEDUPLICATION (AUTHORITATIVE)
# =========================
dedup = merged.unique(
    subset=["drug_name", "drug_id", "disease"],
    keep="first",
)

# =========================
# Save
# =========================
out_file = "ALL_SOURCES_drug_disease_deduplicated.csv"
dedup.write_csv(out_file)

print(f"[DONE] Rows after deduplication: {dedup.height:,}")
print(f"[SAVED] {out_file}")