Unit 1: Feature engineering in Python

pythonimport pandas as pd
from sklearn.preprocessing import MinMaxScaler
df = pd.read_csv("data.csv")
df = df.drop_duplicates()
df["age"] = df["age"].fillna(df["age"].median())
df["city"] = df["city"].fillna("Unknown")
df = df[df["age"].between(0, 100)]            # remove invalid entries
df[["age", "income"]] = MinMaxScaler().fit_transform(df[["age", "income"]])

pythonimport matplotlib.pyplot as plt, seaborn as sns
df["income"].hist(bins=20)
sns.boxplot(x=df["income"])                  # shows outliers
sns.heatmap(df.corr(numeric_only=True), annot=True)
plt.show()

pythonimport numpy as np
from sklearn.preprocessing import PolynomialFeatures
df["age_group"] = pd.cut(df["age"], bins=[0, 18, 35, 60, 100],
                         labels=["teen", "young", "middle", "senior"])
df["log_income"] = np.log1p(df["income"])
df = pd.get_dummies(df, columns=["city"])    # one-hot encoding
poly = PolynomialFeatures(degree=2, include_bias=False)
X_poly = poly.fit_transform(df[["x1", "x2"]])

• Text: tokenise (split into words), remove stop words, stem (running → run) or lemmatise (better → good), then represent with Bag-of-Words (CountVectorizer) or TF-IDF (TfidfVectorizer).
• Images: resize to a fixed size, normalise pixel values to 0–1, and augment with rotation, flips and translation to create more training examples.

pythonfrom statsmodels.tsa.seasonal import seasonal_decompose
result = seasonal_decompose(sales, model="additive", period=12)
result.plot()                                # trend, seasonal, residual

from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
X = StandardScaler().fit_transform(features)
pca = PCA(n_components=2)
X2 = pca.fit_transform(X)
print(pca.explained_variance_ratio_)