https://donml.io/category/pandas/ My Machine Learning Journey Mon, 05 Dec 2022 22:50:45 +0000 en-US hourly 1 https://wordpress.org/?v=6.5.5 https://donml.io/wp-content/uploads/2022/09/cropped-logo-32x32.png https://donml.io/category/pandas/ 32 32 https://donml.io/2022/09/14/a-beginners-pandas-introduction/ https://donml.io/2022/09/14/a-beginners-pandas-introduction/#respond Wed, 14 Sep 2022 17:29:40 +0000 https://donml.io/?p=1142 Introduction to Pandas Machine Learning Zoomcamp¶ 1.9 Introduction to Pandas¶ Plan: Data Frames Series Index Accessing elements Element-wise operations Filtering String operations Summarazing operations Missing values Grouping Getting the NumPy arrays In [1]: import numpy as np import pandas as pd Data Frames¶ In [11]: data = [ ['Nissan', 'Stanza', 1991, 138, 4, 'MANUAL', 'sedan', 2000], ['Hyundai', […]

The post A Beginners Pandas Introduction appeared first on Don's Machine Learning.

]]>

Machine Learning Zoomcamp

1.9 Introduction to Pandas

Plan:

• Data Frames
• Series
• Index
• Accessing elements
• Element-wise operations
• Filtering
• String operations
• Summarazing operations
• Missing values
• Grouping
• Getting the NumPy arrays

import numpy as np
import pandas as pd

Data Frames

data = [
    ['Nissan', 'Stanza', 1991, 138, 4, 'MANUAL', 'sedan', 2000],
    ['Hyundai', 'Sonata', 2017, None, 4, 'AUTOMATIC', 'Sedan', 27150],
    ['Lotus', 'Elise', 2010, 218, 4, 'MANUAL', 'convertible', 54990],
    ['GMC', 'Acadia',  2017, 194, 4, 'AUTOMATIC', '4dr SUV', 34450],
    ['Nissan', 'Frontier', 2017, 261, 6, 'MANUAL', 'Pickup', 32340],
]
columns = [
    'Make', 'Model', 'Year', 'Engine HP', 'Engine Cylinders',
    'Transmission Type', 'Vehicle_Style', 'MSRP'
]

df = pd.DataFrame(data, columns=columns)

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
3	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
4	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

Alternatively, we can use a list of dictionaries to create a dataframe:

data = [
    {
        "Make": "Nissan",
        "Model": "Stanza",
        "Year": 1991,
        "Engine HP": 138.0,
        "Engine Cylinders": 4,
        "Transmission Type": "MANUAL",
        "Vehicle_Style": "sedan",
        "MSRP": 2000
    },
    {
        "Make": "Hyundai",
        "Model": "Sonata",
        "Year": 2017,
        "Engine HP": None,
        "Engine Cylinders": 4,
        "Transmission Type": "AUTOMATIC",
        "Vehicle_Style": "Sedan",
        "MSRP": 27150
    },
    {
        "Make": "Lotus",
        "Model": "Elise",
        "Year": 2010,
        "Engine HP": 218.0,
        "Engine Cylinders": 4,
        "Transmission Type": "MANUAL",
        "Vehicle_Style": "convertible",
        "MSRP": 54990
    },
    {
        "Make": "GMC",
        "Model": "Acadia",
        "Year": 2017,
        "Engine HP": 194.0,
        "Engine Cylinders": 4,
        "Transmission Type": "AUTOMATIC",
        "Vehicle_Style": "4dr SUV",
        "MSRP": 34450
    },
    {
        "Make": "Nissan",
        "Model": "Frontier",
        "Year": 2017,
        "Engine HP": 261.0,
        "Engine Cylinders": 6,
        "Transmission Type": "MANUAL",
        "Vehicle_Style": "Pickup",
        "MSRP": 32340
    }
]

df2 = pd.DataFrame(data)

df2

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
3	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
4	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df.head(2)

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150

Series

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
3	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
4	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df.Make #This does not work with column names with spaces

0     Nissan
1    Hyundai
2      Lotus
3        GMC
4     Nissan
Name: Make, dtype: object

df['Make']

0     Nissan
1    Hyundai
2      Lotus
3        GMC
4     Nissan
Name: Make, dtype: object

df[['Make', 'Model', 'MSRP']]

	Make	Model	MSRP
0	Nissan	Stanza	2000
1	Hyundai	Sonata	27150
2	Lotus	Elise	54990
3	GMC	Acadia	34450
4	Nissan	Frontier	32340

df['id'] = [1, 2, 3, 4, 5]

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP	id
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000	1
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150	2
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990	3
3	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450	4
4	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340	5

del df['id']

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
3	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
4	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

Index

df.index

RangeIndex(start=0, stop=5, step=1)

df.Make

0     Nissan
1    Hyundai
2      Lotus
3        GMC
4     Nissan
Name: Make, dtype: object

df.Make.index

RangeIndex(start=0, stop=5, step=1)

df.loc[1] #Row

Make                   Hyundai
Model                   Sonata
Year                      2017
Engine HP                  NaN
Engine Cylinders             4
Transmission Type    AUTOMATIC
Vehicle_Style            Sedan
MSRP                     27150
Name: 1, dtype: object

df.loc[[1, 2]]

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990

df.index = ['a', 'b', 'c', 'd', 'e']

df.loc[['b', 'c']]

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990

df.iloc[1]

Make                   Hyundai
Model                   Sonata
Year                      2017
Engine HP                  NaN
Engine Cylinders             4
Transmission Type    AUTOMATIC
Vehicle_Style            Sedan
MSRP                     27150
Name: b, dtype: object

df.reset_index()

	index	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
3	d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
4	e	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df.reset_index(drop=True) #Creates new dataframe

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
3	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
4	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df #df is still with the alphabet index even though we reset it

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
e	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df3 = df.reset_index(drop=True) #Assign it to a new variable to keep

df3

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
0	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
1	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
2	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
3	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
4	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

Accessing elements

# was done above

Element-wise operations

df['Engine HP']

a    138.0
b      NaN
c    218.0
d    194.0
e    261.0
Name: Engine HP, dtype: float64

df['Engine HP'] / 100

a    1.38
b     NaN
c    2.18
d    1.94
e    2.61
Name: Engine HP, dtype: float64

df['Engine HP'] * 2

a    276.0
b      NaN
c    436.0
d    388.0
e    522.0
Name: Engine HP, dtype: float64

df['Year'] > 2015

a    False
b     True
c    False
d     True
e     True
Name: Year, dtype: bool

Filtering

df[
    df['Year']  >= 2015
]

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
e	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df[
    df['Make'] == 'Nissan'
]

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
e	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df[
    (df['Make'] == 'Nissan') & (df['Year'] >= 2015)
]

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
e	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

String operations

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
e	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df['Vehicle_Style']

a          sedan
b          Sedan
c    convertible
d        4dr SUV
e         Pickup
Name: Vehicle_Style, dtype: object

df['Vehicle_Style'].str.lower()

a          sedan
b          sedan
c    convertible
d        4dr suv
e         pickup
Name: Vehicle_Style, dtype: object

df['Vehicle_Style'].str.replace(' ', '_')

a          sedan
b          Sedan
c    convertible
d        4dr_SUV
e         Pickup
Name: Vehicle_Style, dtype: object

df['Vehicle_Style'].str.replace(' ', '_').str.lower()

a          sedan
b          sedan
c    convertible
d        4dr_suv
e         pickup
Name: Vehicle_Style, dtype: object

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	Sedan	27150
c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr SUV	34450
e	Nissan	Frontier	2017	261.0	6	MANUAL	Pickup	32340

df['Vehicle_Style']=df['Vehicle_Style'].str.replace(' ', '_').str.lower()

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	sedan	27150
c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr_suv	34450
e	Nissan	Frontier	2017	261.0	6	MANUAL	pickup	32340

Summarazing operations

df.MSRP.max()

54990

df.MSRP.min()

2000

df.MSRP.describe()

count        5.000000
mean     30186.000000
std      18985.044904
min       2000.000000
25%      27150.000000
50%      32340.000000
75%      34450.000000
max      54990.000000
Name: MSRP, dtype: float64

df.describe()

	Year	Engine HP	Engine Cylinders	MSRP
count	5.000000	4.00000	5.000000	5.000000
mean	2010.400000	202.75000	4.400000	30186.000000
std	11.260551	51.29896	0.894427	18985.044904
min	1991.000000	138.00000	4.000000	2000.000000
25%	2010.000000	180.00000	4.000000	27150.000000
50%	2017.000000	206.00000	4.000000	32340.000000
75%	2017.000000	228.75000	4.000000	34450.000000
max	2017.000000	261.00000	6.000000	54990.000000

df.describe().round(2)

	Year	Engine HP	Engine Cylinders	MSRP
count	5.00	4.00	5.00	5.00
mean	2010.40	202.75	4.40	30186.00
std	11.26	51.30	0.89	18985.04
min	1991.00	138.00	4.00	2000.00
25%	2010.00	180.00	4.00	27150.00
50%	2017.00	206.00	4.00	32340.00
75%	2017.00	228.75	4.00	34450.00
max	2017.00	261.00	6.00	54990.00

df.Make

a     Nissan
b    Hyundai
c      Lotus
d        GMC
e     Nissan
Name: Make, dtype: object

df.Make.nunique()

4

df.nunique()

Make                 4
Model                5
Year                 3
Engine HP            4
Engine Cylinders     2
Transmission Type    2
Vehicle_Style        4
MSRP                 5
dtype: int64

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	sedan	27150
c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr_suv	34450
e	Nissan	Frontier	2017	261.0	6	MANUAL	pickup	32340

Missing values

df.isnull()

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	False	False	False	False	False	False	False	False
b	False	False	False	True	False	False	False	False
c	False	False	False	False	False	False	False	False
d	False	False	False	False	False	False	False	False
e	False	False	False	False	False	False	False	False

df.isnull().sum()

Make                 0
Model                0
Year                 0
Engine HP            1
Engine Cylinders     0
Transmission Type    0
Vehicle_Style        0
MSRP                 0
dtype: int64

Grouping

SELECT
    transmission_type,
    AVG(MSRP)
FROM
    cars
GROUP BY
    transmission_type

df.groupby('Transmission Type').MSRP.mean()

Transmission Type
AUTOMATIC    30800.000000
MANUAL       29776.666667
Name: MSRP, dtype: float64

df.groupby('Transmission Type').MSRP.min()

Transmission Type
AUTOMATIC    27150
MANUAL        2000
Name: MSRP, dtype: int64

df.groupby('Transmission Type').MSRP.max()

Transmission Type
AUTOMATIC    34450
MANUAL       54990
Name: MSRP, dtype: int64

Getting the NumPy arrays

df.MSRP.values

array([ 2000, 27150, 54990, 34450, 32340])

df

	Make	Model	Year	Engine HP	Engine Cylinders	Transmission Type	Vehicle_Style	MSRP
a	Nissan	Stanza	1991	138.0	4	MANUAL	sedan	2000
b	Hyundai	Sonata	2017	NaN	4	AUTOMATIC	sedan	27150
c	Lotus	Elise	2010	218.0	4	MANUAL	convertible	54990
d	GMC	Acadia	2017	194.0	4	AUTOMATIC	4dr_suv	34450
e	Nissan	Frontier	2017	261.0	6	MANUAL	pickup	32340

df.to_dict(orient='records')

[{'Make': 'Nissan',
  'Model': 'Stanza',
  'Year': 1991,
  'Engine HP': 138.0,
  'Engine Cylinders': 4,
  'Transmission Type': 'MANUAL',
  'Vehicle_Style': 'sedan',
  'MSRP': 2000},
 {'Make': 'Hyundai',
  'Model': 'Sonata',
  'Year': 2017,
  'Engine HP': nan,
  'Engine Cylinders': 4,
  'Transmission Type': 'AUTOMATIC',
  'Vehicle_Style': 'sedan',
  'MSRP': 27150},
 {'Make': 'Lotus',
  'Model': 'Elise',
  'Year': 2010,
  'Engine HP': 218.0,
  'Engine Cylinders': 4,
  'Transmission Type': 'MANUAL',
  'Vehicle_Style': 'convertible',
  'MSRP': 54990},
 {'Make': 'GMC',
  'Model': 'Acadia',
  'Year': 2017,
  'Engine HP': 194.0,
  'Engine Cylinders': 4,
  'Transmission Type': 'AUTOMATIC',
  'Vehicle_Style': '4dr_suv',
  'MSRP': 34450},
 {'Make': 'Nissan',
  'Model': 'Frontier',
  'Year': 2017,
  'Engine HP': 261.0,
  'Engine Cylinders': 6,
  'Transmission Type': 'MANUAL',
  'Vehicle_Style': 'pickup',
  'MSRP': 32340}]

The post A Beginners Pandas Introduction appeared first on Don's Machine Learning.

]]> https://donml.io/2022/09/14/a-beginners-pandas-introduction/feed/ 0