Merging database-style data frames

Inner Join (Default Merge)

# Creating two sample data frames

df1 = pd.DataFrame({

'ID': [1, 2, 3, 4],

'Name': ['Alice', 'Bob', 'Charlie', 'David'],

'Age': [25, 30, 35, 40]

})

df2 = pd.DataFrame({

'ID': [1, 2, 3],

'Salary': [50000, 60000, 70000]

})

# Merging the data frames on the 'ID' column

merged_df = pd.merge(df1, df2, on='ID')

print(merged_df)

Output:

nginx

Copy

ID Name Age Salary

0 1 Alice 25 50000

1 2 Bob 30 60000

2 3 Charlie 35 70000

Left Join
# Merging the data frames with a left join

merged_df_left = pd.merge(df1, df2, on='ID', how='left')

Output:

pgsql

Copy

ID Name Age Salary

0 1 Alice 25 50000.0

1 2 Bob 30 60000.0

2 3 Charlie 35 70000.0

3 4 David 40 NaN

Right Join
# Merging the data frames with a right join

merged_df_right = pd.merge(df1, df2, on='ID', how='right')

print(merged_df_right)

Output:

nginx

Copy

ID Name Age Salary

0 1 Alice 25.0 50000

1 2 Bob 30.0 60000

2 3 Charlie 35.0 70000

Outer Join

# Merging the data frames with an outer join

merged_df_outer = pd.merge(df1, df2, on='ID', how='outer')

print(merged_df_outer)

Output:

pgsql

Copy
 ID Name Age Salary

0 1 Alice 25.0 50000.0

1 2 Bob 30.0 60000.0

2 3 Charlie 35.0 70000.0

3 4 David 40.0 NaN

Merging on Multiple Columns

# Creating two data frames with multiple common columns

df1 = pd.DataFrame({

'ID': [1, 2, 3],

'Department': ['HR', 'Finance', 'IT'],

'Employee': ['Alice', 'Bob', 'Charlie']

})

df2 = pd.DataFrame({

'ID': [1, 2, 3],

'Department': ['HR', 'Finance', 'IT'],

'Salary': [50000, 60000, 70000]

})

# Merging based on both 'ID' and 'Department'

merged_df_multi = pd.merge(df1, df2, on=['ID', 'Department'])

print(merged_df_multi)

Output:

nginx

Copy

ID Department Employee Salary

0 1 HR Alice 50000

1 2 Finance Bob 60000

2 3 IT Charlie 70000
Merging with Different Column Names

# Creating data frames with different column names for the merge

df1 = pd.DataFrame({

'EmployeeID': [1, 2, 3],

'EmployeeName': ['Alice', 'Bob', 'Charlie']

})

df2 = pd.DataFrame({

'ID': [1, 2, 3],

'Salary': [50000, 60000, 70000]

})

# Merging based on different column names

merged_df_diff_names = pd.merge(df1, df2, left_on='EmployeeID', right_on='ID')

print(merged_df_diff_names)

Output:

nginx

Copy

EmployeeID EmployeeName ID Salary

0 1 Alice 1 50000

1 2 Bob 2 60000

2 3 Charlie 3 70000

Concatenating along with an axis

import pandas as pd

# Creating two data frames

df1 = pd.DataFrame({
 'ID': [1, 2, 3],

'Name': ['Alice', 'Bob', 'Charlie']

})

df2 = pd.DataFrame({

'ID': [4, 5, 6],

'Name': ['David', 'Eve', 'Frank']

})

# Concatenating along axis=0 (row-wise)

concatenated_df = pd.concat([df1, df2], axis=0, ignore_index=True)

print(concatenated_df)

Output:

nginx

Copy

ID Name

0 1 Alice

1 2 Bob

2 3 Charlie

3 4 David

4 5 Eve

5 6 Frank

Concatenating Horizontally (Column-wise)

# Creating two data frames with the same index but different columns

df1 = pd.DataFrame({

'ID': [1, 2, 3],

'Name': ['Alice', 'Bob', 'Charlie']

})
df2 = pd.DataFrame({

'Age': [25, 30, 35],

'Salary': [50000, 60000, 70000]

})

# Concatenating along axis=1 (column-wise)

concatenated_df_horizontal = pd.concat([df1, df2], axis=1)

print(concatenated_df_horizontal)

Output:

nginx

Copy

ID Name Age Salary

0 1 Alice 25 50000

1 2 Bob 30 60000

2 3 Charlie 35 70000

Concatenating with Different Indexes

# Creating two data frames with different indexes

df1 = pd.DataFrame({

'ID': [1, 2],

'Name': ['Alice', 'Bob']

}, index=[0, 1])

df2 = pd.DataFrame({

'Age': [25, 30],

'Salary': [50000, 60000]

}, index=[1, 2])

# Concatenating along axis=0 (row-wise), handling different indexes

concatenated_df_diff_index = pd.concat([df1, df2], axis=0, ignore_index=True)

Output:

pgsql

Copy

ID Name Age Salary

0 1 Alice NaN NaN

1 2 Bob 25.0 50000.0

2 1 NaN 30.0 60000.0

3 2 NaN NaN NaN

Concatenating with Keys (Creating a MultiIndex)

# Concatenating with keys to create a hierarchical index

concatenated_df_keys = pd.concat([df1, df2], axis=0, keys=['df1', 'df2'])

print(concatenated_df_keys)

Output:

pgsql

Copy

ID Name Age Salary

df1 0 1 Alice NaN NaN

1 2 Bob 25.0 50000.0

df2 1 1 NaN 30.0 60000.0

2 2 NaN NaN NaN

Concatenating with Mismatched Columns

# Creating two data frames with mismatched columns

df1 = pd.DataFrame({

'ID': [1, 2],

})

df2 = pd.DataFrame({

'Age': [25, 30],

'Salary': [50000, 60000]

})

# Concatenating along axis=1 (column-wise) with mismatched columns

concatenated_df_mismatched = pd.concat([df1, df2], axis=1)

print(concatenated_df_mismatched)

Output:

pgsql

Copy

ID Name Age Salary

0 1 Alice NaN NaN

1 2 Bob 25.0 50000.0

Merging on index
Simple Merge on Index

import pandas as pd

# Creating two data frames with meaningful indexes

df1 = pd.DataFrame({

'Name': ['Alice', 'Bob', 'Charlie'],

'Age': [25, 30, 35]

}, index=['a', 'b', 'c'])

df2 = pd.DataFrame({

'Salary': [50000, 60000, 70000],

'Department': ['HR', 'Finance', 'IT']

# Merging the data frames on the index

merged_df = pd.merge(df1, df2, left_index=True, right_index=True)

print(merged_df)

Output:

css

Copy

Name Age Salary Department

a Alice 25 50000 HR

b Bob 30 60000 Finance

c Charlie 35 70000 IT

Merge on Index with Different Column Names

# Creating two data frames with different column names but same index

df1 = pd.DataFrame({

'Name': ['Alice', 'Bob', 'Charlie'],

'Age': [25, 30, 35]

}, index=['a', 'b', 'c'])

df2 = pd.DataFrame({

'Salary': [50000, 60000, 70000],

'Department': ['HR', 'Finance', 'IT']

}, index=['a', 'b', 'c'])

# Merging the data frames on index

merged_df_diff_columns = pd.merge(df1, df2, left_index=True, right_index=True)

print(merged_df_diff_columns)

Output:

css
Copy

Name Age Salary Department

a Alice 25 50000 HR

b Bob 30 60000 Finance

c Charlie 35 70000 IT

Merge on Index with how Parameter

# Creating two data frames with different indexes

df1 = pd.DataFrame({

'Name': ['Alice', 'Bob', 'Charlie'],

'Age': [25, 30, 35]

}, index=['a', 'b', 'c'])

df2 = pd.DataFrame({

'Salary': [50000, 60000],

'Department': ['HR', 'Finance']

}, index=['a', 'b'])

# Merging with 'left' join on the index

merged_left = pd.merge(df1, df2, left_index=True, right_index=True, how='left')

print(merged_left)

Output:

Copy

Name Age Salary Department

a Alice 25 50000 HR

b Bob 30 60000 Finance

c Charlie 35 NaN NaN

Merge with outer Join on Index

merged_outer = pd.merge(df1, df2, left_index=True, right_index=True, how='outer')

print(merged_outer)

Output:

Copy

Name Age Salary Department

a Alice 25 50000 HR

b Bob 30 60000 Finance

c Charlie 35 NaN NaN

Merge on Index and Column (Multi-key Merge)

# Creating two data frames with different columns and indexes

df1 = pd.DataFrame({

'Name': ['Alice', 'Bob', 'Charlie'],

'Age': [25, 30, 35]

}, index=['a', 'b', 'c'])

df2 = pd.DataFrame({

'Salary': [50000, 60000, 70000],

'Department': ['HR', 'Finance', 'IT'],

'Age': [25, 30, 35]

}, index=['a', 'b', 'c'])

# Merging on both index and a column

merged_df = pd.merge(df1, df2, left_index=True, right_index=True, on='Age')

print(merged_df)

Output:
css

Copy

Name Age Salary Department

a Alice 25 50000 HR

b Bob 30 60000 Finance

c Charlie 35 70000 IT

Reshaping and pivoting

import pandas as pd

# Creating a sample data frame

df = pd.DataFrame({

'Date': ['2025-03-01', '2025-03-01', '2025-03-02', '2025-03-02'],

'City': ['New York', 'Los Angeles', 'New York', 'Los Angeles'],

'Temperature': [58, 70, 60, 72]

})

# Pivoting the data: Dates as rows, cities as columns, and Temperature as values

pivoted_df = df.pivot(index='Date', columns='City', values='Temperature')

print(pivoted_df)

Output:

sql

Copy

City Los Angeles New York

Date

2025-03-01 70 58

2025-03-02 72 60

Melting DataFrames with melt()

df_wide = pd.DataFrame({

'Date': ['2025-03-01', '2025-03-02'],

'New York': [58, 60],

'Los Angeles': [70, 72]

})

# Melting the data: Convert cities from columns to a single "City" column

melted_df = pd.melt(df_wide, id_vars=['Date'], var_name='City', value_name='Temperature')

print(melted_df)

Output:

yaml

Copy

Date City Temperature

0 2025-03-01 New York 58

1 2025-03-02 New York 60

2 2025-03-01 Los Angeles 70

3 2025-03-02 Los Angeles 72

Stacking DataFrame

# Creating a sample DataFrame

df = pd.DataFrame({

'City': ['New York', 'Los Angeles', 'Chicago'],

'Population': [8175133, 3792621, 2695598],

'Area': [789, 503, 589]

})

# Setting 'City' as the index

df.set_index('City', inplace=True)
# Stacking the DataFrame: Converts columns into a MultiIndex (rows)

stacked_df = df.stack()

print(stacked_df)

Output:

mathematica

Copy

City

New York Population 8175133

Area 789

Los Angeles Population 3792621

Area 503

Chicago Population 2695598

Area 589

dtype: int64

 stack(): Converts columns into rows, resulting in a hierarchical index.

Unstacking DataFrame

# Unstacking the stacked data: Converts rows back to columns

unstacked_df = stacked_df.unstack()

print(unstacked_df)

Output:

sql

Copy

City Population Area

New York 8175133 789

Los Angeles 3792621 503

Chicago 2695598 589

# Creating a sample DataFrame

df = pd.DataFrame({

'Date': ['2025-03-01', '2025-03-01', '2025-03-02', '2025-03-02'],

'City': ['New York', 'Los Angeles', 'New York', 'Los Angeles'],

'Temperature': [58, 70, 60, 72],

'Humidity': [60, 50, 65, 55]

})

# Using pivot_table to reshape and calculate the average Temperature per Date and City

pivot_table_df = df.pivot_table(index='Date', columns='City', values='Temperature',

print(pivot_table_df)

Output:

sql

Copy

City Los Angeles New York

Date

2025-03-01 70 58

2025-03-02 72 60

Handling Missing Data with pivot_table()

# Creating a sample DataFrame with missing values

df_with_missing = pd.DataFrame({

'Date': ['2025-03-01', '2025-03-01', '2025-03-02'],

'City': ['New York', 'Los Angeles', 'New York'],

'Temperature': [58, 70, None],

'Humidity': [60, 50, 65]

# Pivoting with missing data and using mean as aggregation function

pivot_table_missing = df_with_missing.pivot_table(index='Date', columns='City',

print(pivot_table_missing)

Output:

pgsql

Copy

City Los Angeles New York

Date

2025-03-01 70.0 58.0

2025-03-02 NaN NaN