Do “Healthy” Categories Experience More Inflation Than “Unhealthy/Processed” Categories?

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
import sys
sys.path.append('..')  # Just add the parent directory for utils
from utils.data_loader import load_inflation_data

sns.set_style("whitegrid")
plt.rcParams['figure.figsize'] = (14, 6)

cpi_df, _ = load_inflation_data()

df = cpi_df.copy()
df = df.fillna(np.nan)
df = df.dropna(how='all')

print(f"Shape: {df.shape}")
print(f"\nRemaining missing values:")
print(f"{df.isnull().sum()}")

healthy_categories = [
    'Beef_and_veal',
    'Dairy_products',
    'Eggs',
    'Fish_and_seafood',
    'Fresh_fruits',
    'Fresh_fruits_and_vegetables',
    'Fresh_vegetables',
    'Fruits_and_vegetables',
    'Meats',
    'Meats_poultry_and_fish',
    'Other_meats',
    'Pork',
    'Poultry'
]
unhealthy_categories = [
    'Cereals_and_bakery_products',
    'Sugar_and_sweets',
    'Fats_and_oils',
    'Nonalcoholic_beverages'
]

df['Healthy_Avg_Inflation'] = df[healthy_categories].mean(axis=1)
df['Unhealthy_Avg_Inflation'] = df[unhealthy_categories].mean(axis=1)
inflation_summary = df[['Year', 'Healthy_Avg_Inflation', 'Unhealthy_Avg_Inflation']].copy()
for col in healthy_categories:
    inflation_summary[col] = df[col]
for col in unhealthy_categories:
    inflation_summary[col] = df[col]
for col in healthy_categories:
    plt.plot(inflation_summary['Year'], inflation_summary[col],
             color='green', alpha=0.6)
for col in unhealthy_categories:
    plt.plot(inflation_summary['Year'], inflation_summary[col],
             color='red', alpha=0.6)
plt.title('Inflation Rate: Healthy vs. Unhealthy Items')
plt.xlabel('Year')
plt.ylabel('Inflation Rate')
plt.tight_layout()
plt.savefig('../figures/inflation_healthy_vs_unhealthy.png')
plt.show()

healthy_inflation_rates = []
for col in healthy_categories:
    healthy_inflation_rates.extend(df[col].dropna().tolist())

unhealthy_inflation_rates = []
for col in unhealthy_categories:
    unhealthy_inflation_rates.extend(df[col].dropna().tolist())

print("SUMMARY STATISTICS: HEALTHY vs UNHEALTHY CATEGORIES")

summary_stats = pd.DataFrame({
    'Metric': ['Count', 'Mean (%)', 'Median (%)', 'Std Dev (%)', 'Min (%)', 'Max (%)', 'Q1 (%)', 'Q3 (%)'],
    'Healthy Categories': [
        len(healthy_inflation_rates),
        np.mean(healthy_inflation_rates),
        np.median(healthy_inflation_rates),
        np.std(healthy_inflation_rates),
        np.min(healthy_inflation_rates),
        np.max(healthy_inflation_rates),
        np.percentile(healthy_inflation_rates, 25),
        np.percentile(healthy_inflation_rates, 75)
    ],
    'Unhealthy Categories': [
        len(unhealthy_inflation_rates),
        np.mean(unhealthy_inflation_rates),
        np.median(unhealthy_inflation_rates),
        np.std(unhealthy_inflation_rates),
        np.min(unhealthy_inflation_rates),
        np.max(unhealthy_inflation_rates),
        np.percentile(unhealthy_inflation_rates, 25),
        np.percentile(unhealthy_inflation_rates, 75)
    ]
})

print(summary_stats.to_string(index=False))

print("DIFFERENCE (Healthy - Unhealthy)")
diff_mean = np.mean(healthy_inflation_rates) - np.mean(unhealthy_inflation_rates)
diff_median = np.median(healthy_inflation_rates) - np.median(unhealthy_inflation_rates)
print(f"Mean difference: {diff_mean:.2f}%")
print(f"Median difference: {diff_median:.2f}%")

year_comparison = df[['Year', 'Healthy_Avg_Inflation', 'Unhealthy_Avg_Inflation']].copy()
year_comparison['Difference (H-U)'] = year_comparison['Healthy_Avg_Inflation'] - year_comparison['Unhealthy_Avg_Inflation']
year_comparison.columns = ['Year', 'Healthy (%)', 'Unhealthy (%)', 'Difference (%)']

print("\nYear-by-Year Comparison (First 15 Years):")
print(year_comparison.head(15).to_string(index=False))

print("\nYear-by-Year Comparison (Last 15 Years):")
print(year_comparison.tail(15).to_string(index=False))

fig, ax = plt.subplots()

ax.plot(df['Year'], df['Healthy_Avg_Inflation'],
        label='Healthy Categories', color='green', alpha=0.7)
ax.plot(df['Year'], df['Unhealthy_Avg_Inflation'],
        label='Unhealthy/Processed Categories', color='red', alpha=0.7)

ax.axhline(y=0, color='black')
ax.set_xlabel('Year')
ax.set_ylabel('Inflation Rate (%)')
ax.set_title('Year-over-Year Inflation Rates: Healthy vs Unhealthy Food Categories (1974-2024)')
ax.legend()
plt.savefig('../figures/yoy_inflation_healthy_vs_unhealthy.png')
plt.show()

fig, ax = plt.subplots()

sns.boxplot(data=[healthy_inflation_rates, unhealthy_inflation_rates], 
            palette=['green', 'red'])

plt.xticks([0, 1], ['Healthy Categories', 'Unhealthy Categories'])
plt.ylabel('Inflation Rate (%)')
plt.title('Distribution of Inflation Rates')
plt.savefig('../figures/healthy_vs_unhealthy_inflation_boxplot.png')
plt.show()

fig, axes = plt.subplots(1, 2)

axes[0].hist(healthy_inflation_rates, color='green', edgecolor='black', bins=30, density='True')
axes[0].set_xlabel('Inflation Rate (%)')
axes[0].set_ylabel('Frequency')
axes[0].set_title('Distribution: Healthy Categories')
axes[0].axvline(np.mean(healthy_inflation_rates), color='darkgreen', linestyle='--')

axes[1].hist(unhealthy_inflation_rates, color='red', edgecolor='black', bins=30, density='True')
axes[1].set_xlabel('Inflation Rate (%)')
axes[1].set_ylabel('Frequency')
axes[1].set_title('Distribution: Unhealthy/Processed Categories')
axes[1].axvline(np.mean(unhealthy_inflation_rates), color='darkred', linestyle='--')

plt.tight_layout()
plt.savefig('../figures/healthy_vs_unhealthy_inflation_histogram.png')
plt.show()

print("STATISTICAL TEST 1: Independent Samples t-test")
print("Null Hypothesis (H0): There is no significant difference in mean")
print("inflation between healthy and unhealthy categories")
print("Alternative Hypothesis (H1): There is a significant difference\n")

t_stat, t_pvalue = stats.ttest_ind(healthy_inflation_rates, unhealthy_inflation_rates)
print(f"t-statistic: {t_stat:.4f}")
print(f"p-value: {t_pvalue:.6f}")
print(f"Result: FAIL TO REJECT the null hypothesis (p ≥ 0.05)")
print("Conclusion: There is NO statistically significant difference in inflation")
print("rates between healthy and unhealthy food categories.")
    
print(f"\nEFFECT SIZE: Cohen's d")
n1, n2 = len(healthy_inflation_rates), len(unhealthy_inflation_rates)
var1, var2 = np.var(healthy_inflation_rates, ddof=1), np.var(unhealthy_inflation_rates, ddof=1)
pooled_std = np.sqrt(((n1-1)*var1 + (n2-1)*var2) / (n1 + n2 - 2))
cohens_d = (np.mean(healthy_inflation_rates) - np.mean(unhealthy_inflation_rates)) / pooled_std

if abs(cohens_d) < 0.2:
    effect_interpretation = "negligible"
elif abs(cohens_d) < 0.5:
    effect_interpretation = "small"
elif abs(cohens_d) < 0.8:
    effect_interpretation = "medium"
else:
    effect_interpretation = "large"

print(f"Cohen's d: {cohens_d:.4f}")
print(f"Cohen's d effect size is {effect_interpretation}")

print("STATISTICAL TEST 2: Mann-Whitney U Test (Non-parametric)")
print("This test doesn't assume normal distribution of data\n")

u_stat, u_pvalue = stats.mannwhitneyu(healthy_inflation_rates, unhealthy_inflation_rates)
print(f"U-statistic: {u_stat:.4f}")
print(f"p-value: {u_pvalue:.6f}")

print(f"Result: FAIL TO REJECT the null hypothesis (p ≥ 0.05)")
print("Conclusion: There is NO statistically significant difference in inflation")
print("distributions between healthy and unhealthy categories.")

print("CONFIDENCE INTERVALS (95%)")

healthy_mean = np.mean(healthy_inflation_rates)
healthy_sem = stats.sem(healthy_inflation_rates)
healthy_ci = stats.t.interval(0.95, len(healthy_inflation_rates)-1, loc=healthy_mean, scale=healthy_sem)

print("Healthy Categories:")
print(f"  Mean: {healthy_mean:.4f}%")
print(f"  95% CI: [{healthy_ci[0]:.4f}%, {healthy_ci[1]:.4f}%]")

unhealthy_mean = np.mean(unhealthy_inflation_rates)
unhealthy_sem = stats.sem(unhealthy_inflation_rates)
unhealthy_ci = stats.t.interval(0.95, len(unhealthy_inflation_rates)-1, loc=unhealthy_mean, scale=unhealthy_sem)

print(f"\nUnhealthy/Processed Categories:")
print(f"  Mean: {unhealthy_mean:.4f}%")
print(f"  95% CI: [{unhealthy_ci[0]:.4f}%, {unhealthy_ci[1]:.4f}%]")

print(f"\nDifference (Healthy - Unhealthy): {healthy_mean - unhealthy_mean:.4f}%")

print("SUMMARY OF FINDINGS")

healthy_mean = np.mean(healthy_inflation_rates)
unhealthy_mean = np.mean(unhealthy_inflation_rates)
difference = healthy_mean - unhealthy_mean

print(f"\n1. AVERAGE INFLATION RATES (1974-2022):")
print(f"   - Healthy Categories: {healthy_mean:.2f}%")
print(f"   - Unhealthy/Processed Categories: {unhealthy_mean:.2f}%")
print(f"   - Difference: {difference:.2f}%")
print(f"   - Unhealthy/processed categories experienced HIGHER inflation")

print(f"\n2. VARIABILITY (Standard Deviation):")
print(f"   - Healthy Categories: {np.std(healthy_inflation_rates):.2f}%")
print(f"   - Unhealthy/Processed Categories: {np.std(unhealthy_inflation_rates):.2f}%")

print(f"\n3. STATISTICAL SIGNIFICANCE:")
print(f"   - t-test p-value: {t_pvalue:.6f} (p ≥ 0.05)")
print(f"   - Result: The difference is NOT statistically significant")

print(f"\n4. EFFECT SIZE:")
print(f"   - Cohen's d: {cohens_d:.4f} ({effect_interpretation} effect)")

print("CONCLUSION")
print(f"""
Based on the analysis of USDA Consumer Price Index (CPI) data from 1974-2022:

The data shows that over the nearly 50-year period, HEALTHY food categories 
(fruits, vegetables, dairy, cereals) have experienced {abs(difference):.2f}% 
LOWER average inflation compared to unhealthy/processed 
categories (sugars, fats & oils, processed foods).

Statistical Testing Results:
- The t-test p-value of {t_pvalue:.6f} indicates that the difference is 
  NOT statistically significant.
  
- The effect size (Cohen's d = {cohens_d:.4f}) is negligible.

This suggests that unhealthy food prices have inflated more rapidly than
processed/unhealthy food prices over the study period, which has important 
implications for food affordability and dietary choices.
""")