diff --git a/doc/source/release.rst b/doc/source/release.rst
index bd3fbf53de039..8ba0574df97cb 100644
--- a/doc/source/release.rst
+++ b/doc/source/release.rst
@@ -234,6 +234,7 @@ API Changes
     Indexing on other index types are preserved (and positional fallback for ``[],ix``), with the exception, that floating point slicing
     on indexes on non ``Float64Index`` will raise a ``TypeError``, e.g. ``Series(range(5))[3.5:4.5]`` (:issue:`263`)
   - Make Categorical repr nicer (:issue:`4368`)
+  - Remove deprecated ``Factor`` (:issue:`3650`)
 
 Internal Refactoring
 ~~~~~~~~~~~~~~~~~~~~
diff --git a/pandas/core/api.py b/pandas/core/api.py
index b4afe90d46842..2b4063eae1f74 100644
--- a/pandas/core/api.py
+++ b/pandas/core/api.py
@@ -5,7 +5,7 @@
 
 from pandas.core.algorithms import factorize, match, unique, value_counts
 from pandas.core.common import isnull, notnull
-from pandas.core.categorical import Categorical, Factor
+from pandas.core.categorical import Categorical
 from pandas.core.format import (set_printoptions, reset_printoptions,
                                 set_eng_float_format)
 from pandas.core.index import Index, Int64Index, Float64Index, MultiIndex
diff --git a/pandas/core/categorical.py b/pandas/core/categorical.py
index 97ed0fdb0da30..f412947f92255 100644
--- a/pandas/core/categorical.py
+++ b/pandas/core/categorical.py
@@ -230,16 +230,3 @@ def describe(self):
                                     counts=counts,
                                     freqs=freqs,
                                     levels=self.levels)).set_index('levels')
-
-
-class Factor(Categorical):
-    def __init__(self, labels, levels=None, name=None):
-        from warnings import warn
-        warn("Factor is deprecated. Use Categorical instead", FutureWarning)
-        super(Factor, self).__init__(labels, levels, name)
-
-    @classmethod
-    def from_array(cls, data):
-        from warnings import warn
-        warn("Factor is deprecated. Use Categorical instead", FutureWarning)
-        return super(Factor, cls).from_array(data)