]> nv-tegra.nvidia Code Review - linux-2.6.git/blobdiff - drivers/pci/pci-driver.c
Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux...
[linux-2.6.git] / drivers / pci / pci-driver.c
index 4884c4840b3d317bca74b9d5d76a2c7978dfee52..135df164a4c1e897e4d36a7d5a49336518669480 100644 (file)
 #include <linux/slab.h>
 #include <linux/sched.h>
 #include <linux/cpu.h>
+#include <linux/pm_runtime.h>
 #include "pci.h"
 
-/*
- * Dynamic device IDs are disabled for !CONFIG_HOTPLUG
- */
-
 struct pci_dynid {
        struct list_head node;
        struct pci_device_id id;
 };
 
-#ifdef CONFIG_HOTPLUG
+/**
+ * pci_add_dynid - add a new PCI device ID to this driver and re-probe devices
+ * @drv: target pci driver
+ * @vendor: PCI vendor ID
+ * @device: PCI device ID
+ * @subvendor: PCI subvendor ID
+ * @subdevice: PCI subdevice ID
+ * @class: PCI class
+ * @class_mask: PCI class mask
+ * @driver_data: private driver data
+ *
+ * Adds a new dynamic pci device ID to this driver and causes the
+ * driver to probe for all devices again.  @drv must have been
+ * registered prior to calling this function.
+ *
+ * CONTEXT:
+ * Does GFP_KERNEL allocation.
+ *
+ * RETURNS:
+ * 0 on success, -errno on failure.
+ */
+int pci_add_dynid(struct pci_driver *drv,
+                 unsigned int vendor, unsigned int device,
+                 unsigned int subvendor, unsigned int subdevice,
+                 unsigned int class, unsigned int class_mask,
+                 unsigned long driver_data)
+{
+       struct pci_dynid *dynid;
+       int retval;
+
+       dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
+       if (!dynid)
+               return -ENOMEM;
+
+       dynid->id.vendor = vendor;
+       dynid->id.device = device;
+       dynid->id.subvendor = subvendor;
+       dynid->id.subdevice = subdevice;
+       dynid->id.class = class;
+       dynid->id.class_mask = class_mask;
+       dynid->id.driver_data = driver_data;
+
+       spin_lock(&drv->dynids.lock);
+       list_add_tail(&dynid->node, &drv->dynids.list);
+       spin_unlock(&drv->dynids.lock);
+
+       get_driver(&drv->driver);
+       retval = driver_attach(&drv->driver);
+       put_driver(&drv->driver);
+
+       return retval;
+}
+
+static void pci_free_dynids(struct pci_driver *drv)
+{
+       struct pci_dynid *dynid, *n;
 
+       spin_lock(&drv->dynids.lock);
+       list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
+               list_del(&dynid->node);
+               kfree(dynid);
+       }
+       spin_unlock(&drv->dynids.lock);
+}
+
+/*
+ * Dynamic device ID manipulation via sysfs is disabled for !CONFIG_HOTPLUG
+ */
+#ifdef CONFIG_HOTPLUG
 /**
- * store_new_id - add a new PCI device ID to this driver and re-probe devices
+ * store_new_id - sysfs frontend to pci_add_dynid()
  * @driver: target device driver
  * @buf: buffer for scanning device ID data
  * @count: input size
  *
- * Adds a new dynamic pci device ID to this driver,
- * and causes the driver to probe for all devices again.
+ * Allow PCI IDs to be added to an existing driver via sysfs.
  */
 static ssize_t
 store_new_id(struct device_driver *driver, const char *buf, size_t count)
 {
-       struct pci_dynid *dynid;
        struct pci_driver *pdrv = to_pci_driver(driver);
        const struct pci_device_id *ids = pdrv->id_table;
        __u32 vendor, device, subvendor=PCI_ANY_ID,
                subdevice=PCI_ANY_ID, class=0, class_mask=0;
        unsigned long driver_data=0;
        int fields=0;
-       int retval=0;
+       int retval;
 
        fields = sscanf(buf, "%x %x %x %x %x %x %lx",
                        &vendor, &device, &subvendor, &subdevice,
@@ -72,45 +134,59 @@ store_new_id(struct device_driver *driver, const char *buf, size_t count)
                        return retval;
        }
 
-       dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
-       if (!dynid)
-               return -ENOMEM;
-
-       dynid->id.vendor = vendor;
-       dynid->id.device = device;
-       dynid->id.subvendor = subvendor;
-       dynid->id.subdevice = subdevice;
-       dynid->id.class = class;
-       dynid->id.class_mask = class_mask;
-       dynid->id.driver_data = driver_data;
-
-       spin_lock(&pdrv->dynids.lock);
-       list_add_tail(&dynid->node, &pdrv->dynids.list);
-       spin_unlock(&pdrv->dynids.lock);
-
-       if (get_driver(&pdrv->driver)) {
-               retval = driver_attach(&pdrv->driver);
-               put_driver(&pdrv->driver);
-       }
-
+       retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
+                              class, class_mask, driver_data);
        if (retval)
                return retval;
        return count;
 }
 static DRIVER_ATTR(new_id, S_IWUSR, NULL, store_new_id);
 
-static void
-pci_free_dynids(struct pci_driver *drv)
+/**
+ * store_remove_id - remove a PCI device ID from this driver
+ * @driver: target device driver
+ * @buf: buffer for scanning device ID data
+ * @count: input size
+ *
+ * Removes a dynamic pci device ID to this driver.
+ */
+static ssize_t
+store_remove_id(struct device_driver *driver, const char *buf, size_t count)
 {
        struct pci_dynid *dynid, *n;
+       struct pci_driver *pdrv = to_pci_driver(driver);
+       __u32 vendor, device, subvendor = PCI_ANY_ID,
+               subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
+       int fields = 0;
+       int retval = -ENODEV;
 
-       spin_lock(&drv->dynids.lock);
-       list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
-               list_del(&dynid->node);
-               kfree(dynid);
+       fields = sscanf(buf, "%x %x %x %x %x %x",
+                       &vendor, &device, &subvendor, &subdevice,
+                       &class, &class_mask);
+       if (fields < 2)
+               return -EINVAL;
+
+       spin_lock(&pdrv->dynids.lock);
+       list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
+               struct pci_device_id *id = &dynid->id;
+               if ((id->vendor == vendor) &&
+                   (id->device == device) &&
+                   (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
+                   (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
+                   !((id->class ^ class) & class_mask)) {
+                       list_del(&dynid->node);
+                       kfree(dynid);
+                       retval = 0;
+                       break;
+               }
        }
-       spin_unlock(&drv->dynids.lock);
+       spin_unlock(&pdrv->dynids.lock);
+
+       if (retval)
+               return retval;
+       return count;
 }
+static DRIVER_ATTR(remove_id, S_IWUSR, NULL, store_remove_id);
 
 static int
 pci_create_newid_file(struct pci_driver *drv)
@@ -125,13 +201,31 @@ static void pci_remove_newid_file(struct pci_driver *drv)
 {
        driver_remove_file(&drv->driver, &driver_attr_new_id);
 }
+
+static int
+pci_create_removeid_file(struct pci_driver *drv)
+{
+       int error = 0;
+       if (drv->probe != NULL)
+               error = driver_create_file(&drv->driver,&driver_attr_remove_id);
+       return error;
+}
+
+static void pci_remove_removeid_file(struct pci_driver *drv)
+{
+       driver_remove_file(&drv->driver, &driver_attr_remove_id);
+}
 #else /* !CONFIG_HOTPLUG */
-static inline void pci_free_dynids(struct pci_driver *drv) {}
 static inline int pci_create_newid_file(struct pci_driver *drv)
 {
        return 0;
 }
 static inline void pci_remove_newid_file(struct pci_driver *drv) {}
+static inline int pci_create_removeid_file(struct pci_driver *drv)
+{
+       return 0;
+}
+static inline void pci_remove_removeid_file(struct pci_driver *drv) {}
 #endif
 
 /**
@@ -195,8 +289,26 @@ struct drv_dev_and_id {
 static long local_pci_probe(void *_ddi)
 {
        struct drv_dev_and_id *ddi = _ddi;
-
-       return ddi->drv->probe(ddi->dev, ddi->id);
+       struct device *dev = &ddi->dev->dev;
+       int rc;
+
+       /* Unbound PCI devices are always set to disabled and suspended.
+        * During probe, the device is set to enabled and active and the
+        * usage count is incremented.  If the driver supports runtime PM,
+        * it should call pm_runtime_put_noidle() in its probe routine and
+        * pm_runtime_get_noresume() in its remove routine.
+        */
+       pm_runtime_get_noresume(dev);
+       pm_runtime_set_active(dev);
+       pm_runtime_enable(dev);
+
+       rc = ddi->drv->probe(ddi->dev, ddi->id);
+       if (rc) {
+               pm_runtime_disable(dev);
+               pm_runtime_set_suspended(dev);
+               pm_runtime_put_noidle(dev);
+       }
+       return rc;
 }
 
 static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
@@ -212,10 +324,9 @@ static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
        node = dev_to_node(&dev->dev);
        if (node >= 0) {
                int cpu;
-               node_to_cpumask_ptr(nodecpumask, node);
 
                get_online_cpus();
-               cpu = cpumask_any_and(nodecpumask, cpu_online_mask);
+               cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
                if (cpu < nr_cpu_ids)
                        error = work_on_cpu(cpu, local_pci_probe, &ddi);
                else
@@ -227,7 +338,7 @@ static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
 }
 
 /**
- * __pci_device_probe()
+ * __pci_device_probe - check if a driver wants to claim a specific PCI device
  * @drv: driver to call to check if it wants the PCI device
  * @pci_dev: PCI device being probed
  * 
@@ -276,11 +387,19 @@ static int pci_device_remove(struct device * dev)
        struct pci_driver * drv = pci_dev->driver;
 
        if (drv) {
-               if (drv->remove)
+               if (drv->remove) {
+                       pm_runtime_get_sync(dev);
                        drv->remove(pci_dev);
+                       pm_runtime_put_noidle(dev);
+               }
                pci_dev->driver = NULL;
        }
 
+       /* Undo the runtime PM settings in local_pci_probe() */
+       pm_runtime_disable(dev);
+       pm_runtime_set_suspended(dev);
+       pm_runtime_put_noidle(dev);
+
        /*
         * If the device is still on, set the power state as "unknown",
         * since it might change by the next time we load the driver.
@@ -312,6 +431,36 @@ static void pci_device_shutdown(struct device *dev)
        pci_msix_shutdown(pci_dev);
 }
 
+#ifdef CONFIG_PM
+
+/* Auxiliary functions used for system resume and run-time resume. */
+
+/**
+ * pci_restore_standard_config - restore standard config registers of PCI device
+ * @pci_dev: PCI device to handle
+ */
+static int pci_restore_standard_config(struct pci_dev *pci_dev)
+{
+       pci_update_current_state(pci_dev, PCI_UNKNOWN);
+
+       if (pci_dev->current_state != PCI_D0) {
+               int error = pci_set_power_state(pci_dev, PCI_D0);
+               if (error)
+                       return error;
+       }
+
+       pci_restore_state(pci_dev);
+       return 0;
+}
+
+static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
+{
+       pci_restore_standard_config(pci_dev);
+       pci_fixup_device(pci_fixup_resume_early, pci_dev);
+}
+
+#endif
+
 #ifdef CONFIG_PM_SLEEP
 
 /*
@@ -352,47 +501,58 @@ static int pci_legacy_suspend(struct device *dev, pm_message_t state)
 {
        struct pci_dev * pci_dev = to_pci_dev(dev);
        struct pci_driver * drv = pci_dev->driver;
-       int i = 0;
 
        if (drv && drv->suspend) {
-               pci_dev->state_saved = false;
-
-               i = drv->suspend(pci_dev, state);
-               suspend_report_result(drv->suspend, i);
-               if (i)
-                       return i;
-
-               if (pci_dev->state_saved)
-                       goto Fixup;
-
-               if (WARN_ON_ONCE(pci_dev->current_state != PCI_D0))
-                       goto Fixup;
+               pci_power_t prev = pci_dev->current_state;
+               int error;
+
+               error = drv->suspend(pci_dev, state);
+               suspend_report_result(drv->suspend, error);
+               if (error)
+                       return error;
+
+               if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
+                   && pci_dev->current_state != PCI_UNKNOWN) {
+                       WARN_ONCE(pci_dev->current_state != prev,
+                               "PCI PM: Device state not saved by %pF\n",
+                               drv->suspend);
+               }
        }
 
-       pci_save_state(pci_dev);
-       pci_dev->state_saved = true;
-       /*
-        * This is for compatibility with existing code with legacy PM support.
-        */
-       pci_pm_set_unknown_state(pci_dev);
-
- Fixup:
        pci_fixup_device(pci_fixup_suspend, pci_dev);
 
-       return i;
+       return 0;
 }
 
 static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
 {
        struct pci_dev * pci_dev = to_pci_dev(dev);
        struct pci_driver * drv = pci_dev->driver;
-       int i = 0;
 
        if (drv && drv->suspend_late) {
-               i = drv->suspend_late(pci_dev, state);
-               suspend_report_result(drv->suspend_late, i);
+               pci_power_t prev = pci_dev->current_state;
+               int error;
+
+               error = drv->suspend_late(pci_dev, state);
+               suspend_report_result(drv->suspend_late, error);
+               if (error)
+                       return error;
+
+               if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
+                   && pci_dev->current_state != PCI_UNKNOWN) {
+                       WARN_ONCE(pci_dev->current_state != prev,
+                               "PCI PM: Device state not saved by %pF\n",
+                               drv->suspend_late);
+                       return 0;
+               }
        }
-       return i;
+
+       if (!pci_dev->state_saved)
+               pci_save_state(pci_dev);
+
+       pci_pm_set_unknown_state(pci_dev);
+
+       return 0;
 }
 
 static int pci_legacy_resume_early(struct device *dev)
@@ -417,42 +577,19 @@ static int pci_legacy_resume(struct device *dev)
 
 /* Auxiliary functions used by the new power management framework */
 
-static void pci_pm_default_resume_noirq(struct pci_dev *pci_dev)
-{
-       pci_restore_standard_config(pci_dev);
-       pci_dev->state_saved = false;
-       pci_fixup_device(pci_fixup_resume_early, pci_dev);
-}
-
-static int pci_pm_default_resume(struct pci_dev *pci_dev)
+static void pci_pm_default_resume(struct pci_dev *pci_dev)
 {
        pci_fixup_device(pci_fixup_resume, pci_dev);
 
        if (!pci_is_bridge(pci_dev))
                pci_enable_wake(pci_dev, PCI_D0, false);
-
-       return pci_pm_reenable_device(pci_dev);
-}
-
-static void pci_pm_default_suspend_generic(struct pci_dev *pci_dev)
-{
-       /* If device is enabled at this point, disable it */
-       pci_disable_enabled_device(pci_dev);
-       /*
-        * Save state with interrupts enabled, because in principle the bus the
-        * device is on may be put into a low power state after this code runs.
-        */
-       pci_save_state(pci_dev);
 }
 
 static void pci_pm_default_suspend(struct pci_dev *pci_dev)
 {
-       pci_pm_default_suspend_generic(pci_dev);
-
+       /* Disable non-bridge devices without PM support */
        if (!pci_is_bridge(pci_dev))
-               pci_prepare_to_sleep(pci_dev);
-
-       pci_fixup_device(pci_fixup_suspend, pci_dev);
+               pci_disable_enabled_device(pci_dev);
 }
 
 static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
@@ -478,6 +615,17 @@ static int pci_pm_prepare(struct device *dev)
        struct device_driver *drv = dev->driver;
        int error = 0;
 
+       /*
+        * PCI devices suspended at run time need to be resumed at this
+        * point, because in general it is necessary to reconfigure them for
+        * system suspend.  Namely, if the device is supposed to wake up the
+        * system from the sleep state, we may need to reconfigure it for this
+        * purpose.  In turn, if the device is not supposed to wake up the
+        * system from the sleep state, we'll have to prevent it from signaling
+        * wake-up.
+        */
+       pm_runtime_resume(dev);
+
        if (drv && drv->pm && drv->pm->prepare)
                error = drv->pm->prepare(dev);
 
@@ -492,46 +640,91 @@ static void pci_pm_complete(struct device *dev)
                drv->pm->complete(dev);
 }
 
+#else /* !CONFIG_PM_SLEEP */
+
+#define pci_pm_prepare NULL
+#define pci_pm_complete        NULL
+
+#endif /* !CONFIG_PM_SLEEP */
+
 #ifdef CONFIG_SUSPEND
 
 static int pci_pm_suspend(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
-       struct device_driver *drv = dev->driver;
-       int error = 0;
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_suspend(dev, PMSG_SUSPEND);
 
-       if (drv && drv->pm && drv->pm->suspend) {
-               error = drv->pm->suspend(dev);
-               suspend_report_result(drv->pm->suspend, error);
+       if (!pm) {
+               pci_pm_default_suspend(pci_dev);
+               goto Fixup;
        }
 
-       if (!error)
-               pci_pm_default_suspend(pci_dev);
+       if (pm->suspend) {
+               pci_power_t prev = pci_dev->current_state;
+               int error;
 
-       return error;
+               error = pm->suspend(dev);
+               suspend_report_result(pm->suspend, error);
+               if (error)
+                       return error;
+
+               if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
+                   && pci_dev->current_state != PCI_UNKNOWN) {
+                       WARN_ONCE(pci_dev->current_state != prev,
+                               "PCI PM: State of device not saved by %pF\n",
+                               pm->suspend);
+               }
+       }
+
+ Fixup:
+       pci_fixup_device(pci_fixup_suspend, pci_dev);
+
+       return 0;
 }
 
 static int pci_pm_suspend_noirq(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
-       struct device_driver *drv = dev->driver;
-       int error = 0;
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
 
-       if (drv && drv->pm && drv->pm->suspend_noirq) {
-               error = drv->pm->suspend_noirq(dev);
-               suspend_report_result(drv->pm->suspend_noirq, error);
+       if (!pm) {
+               pci_save_state(pci_dev);
+               return 0;
        }
 
-       if (!error)
-               pci_pm_set_unknown_state(pci_dev);
+       if (pm->suspend_noirq) {
+               pci_power_t prev = pci_dev->current_state;
+               int error;
+
+               error = pm->suspend_noirq(dev);
+               suspend_report_result(pm->suspend_noirq, error);
+               if (error)
+                       return error;
+
+               if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
+                   && pci_dev->current_state != PCI_UNKNOWN) {
+                       WARN_ONCE(pci_dev->current_state != prev,
+                               "PCI PM: State of device not saved by %pF\n",
+                               pm->suspend_noirq);
+                       return 0;
+               }
+       }
 
-       return error;
+       if (!pci_dev->state_saved) {
+               pci_save_state(pci_dev);
+               if (!pci_is_bridge(pci_dev))
+                       pci_prepare_to_sleep(pci_dev);
+       }
+
+       pci_pm_set_unknown_state(pci_dev);
+
+       return 0;
 }
 
 static int pci_pm_resume_noirq(struct device *dev)
@@ -540,7 +733,7 @@ static int pci_pm_resume_noirq(struct device *dev)
        struct device_driver *drv = dev->driver;
        int error = 0;
 
-       pci_pm_default_resume_noirq(pci_dev);
+       pci_pm_default_resume_early(pci_dev);
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_resume_early(dev);
@@ -554,7 +747,7 @@ static int pci_pm_resume_noirq(struct device *dev)
 static int pci_pm_resume(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
-       struct device_driver *drv = dev->driver;
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
        int error = 0;
 
        /*
@@ -567,10 +760,14 @@ static int pci_pm_resume(struct device *dev)
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_resume(dev);
 
-       error = pci_pm_default_resume(pci_dev);
+       pci_pm_default_resume(pci_dev);
 
-       if (!error && drv && drv->pm && drv->pm->resume)
-               error = drv->pm->resume(dev);
+       if (pm) {
+               if (pm->resume)
+                       error = pm->resume(dev);
+       } else {
+               pci_pm_reenable_device(pci_dev);
+       }
 
        return error;
 }
@@ -584,46 +781,56 @@ static int pci_pm_resume(struct device *dev)
 
 #endif /* !CONFIG_SUSPEND */
 
-#ifdef CONFIG_HIBERNATION
+#ifdef CONFIG_HIBERNATE_CALLBACKS
 
 static int pci_pm_freeze(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
-       struct device_driver *drv = dev->driver;
-       int error = 0;
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_suspend(dev, PMSG_FREEZE);
 
-       if (drv && drv->pm && drv->pm->freeze) {
-               error = drv->pm->freeze(dev);
-               suspend_report_result(drv->pm->freeze, error);
+       if (!pm) {
+               pci_pm_default_suspend(pci_dev);
+               return 0;
        }
 
-       if (!error)
-               pci_pm_default_suspend_generic(pci_dev);
+       if (pm->freeze) {
+               int error;
 
-       return error;
+               error = pm->freeze(dev);
+               suspend_report_result(pm->freeze, error);
+               if (error)
+                       return error;
+       }
+
+       return 0;
 }
 
 static int pci_pm_freeze_noirq(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
        struct device_driver *drv = dev->driver;
-       int error = 0;
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_suspend_late(dev, PMSG_FREEZE);
 
        if (drv && drv->pm && drv->pm->freeze_noirq) {
+               int error;
+
                error = drv->pm->freeze_noirq(dev);
                suspend_report_result(drv->pm->freeze_noirq, error);
+               if (error)
+                       return error;
        }
 
-       if (!error)
-               pci_pm_set_unknown_state(pci_dev);
+       if (!pci_dev->state_saved)
+               pci_save_state(pci_dev);
 
-       return error;
+       pci_pm_set_unknown_state(pci_dev);
+
+       return 0;
 }
 
 static int pci_pm_thaw_noirq(struct device *dev)
@@ -646,16 +853,20 @@ static int pci_pm_thaw_noirq(struct device *dev)
 static int pci_pm_thaw(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
-       struct device_driver *drv = dev->driver;
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
        int error = 0;
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_resume(dev);
 
-       pci_pm_reenable_device(pci_dev);
+       if (pm) {
+               if (pm->thaw)
+                       error = pm->thaw(dev);
+       } else {
+               pci_pm_reenable_device(pci_dev);
+       }
 
-       if (drv && drv->pm && drv->pm->thaw)
-               error =  drv->pm->thaw(dev);
+       pci_dev->state_saved = false;
 
        return error;
 }
@@ -663,37 +874,55 @@ static int pci_pm_thaw(struct device *dev)
 static int pci_pm_poweroff(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
-       struct device_driver *drv = dev->driver;
-       int error = 0;
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_suspend(dev, PMSG_HIBERNATE);
 
-       if (drv && drv->pm && drv->pm->poweroff) {
-               error = drv->pm->poweroff(dev);
-               suspend_report_result(drv->pm->poweroff, error);
+       if (!pm) {
+               pci_pm_default_suspend(pci_dev);
+               goto Fixup;
        }
 
-       if (!error)
-               pci_pm_default_suspend(pci_dev);
+       if (pm->poweroff) {
+               int error;
 
-       return error;
+               error = pm->poweroff(dev);
+               suspend_report_result(pm->poweroff, error);
+               if (error)
+                       return error;
+       }
+
+ Fixup:
+       pci_fixup_device(pci_fixup_suspend, pci_dev);
+
+       return 0;
 }
 
 static int pci_pm_poweroff_noirq(struct device *dev)
 {
+       struct pci_dev *pci_dev = to_pci_dev(dev);
        struct device_driver *drv = dev->driver;
-       int error = 0;
 
        if (pci_has_legacy_pm_support(to_pci_dev(dev)))
                return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
 
-       if (drv && drv->pm && drv->pm->poweroff_noirq) {
+       if (!drv || !drv->pm)
+               return 0;
+
+       if (drv->pm->poweroff_noirq) {
+               int error;
+
                error = drv->pm->poweroff_noirq(dev);
                suspend_report_result(drv->pm->poweroff_noirq, error);
+               if (error)
+                       return error;
        }
 
-       return error;
+       if (!pci_dev->state_saved && !pci_is_bridge(pci_dev))
+               pci_prepare_to_sleep(pci_dev);
+
+       return 0;
 }
 
 static int pci_pm_restore_noirq(struct device *dev)
@@ -702,7 +931,7 @@ static int pci_pm_restore_noirq(struct device *dev)
        struct device_driver *drv = dev->driver;
        int error = 0;
 
-       pci_pm_default_resume_noirq(pci_dev);
+       pci_pm_default_resume_early(pci_dev);
 
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_resume_early(dev);
@@ -716,7 +945,7 @@ static int pci_pm_restore_noirq(struct device *dev)
 static int pci_pm_restore(struct device *dev)
 {
        struct pci_dev *pci_dev = to_pci_dev(dev);
-       struct device_driver *drv = dev->driver;
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
        int error = 0;
 
        /*
@@ -729,15 +958,19 @@ static int pci_pm_restore(struct device *dev)
        if (pci_has_legacy_pm_support(pci_dev))
                return pci_legacy_resume(dev);
 
-       error = pci_pm_default_resume(pci_dev);
+       pci_pm_default_resume(pci_dev);
 
-       if (!error && drv && drv->pm && drv->pm->restore)
-               error = drv->pm->restore(dev);
+       if (pm) {
+               if (pm->restore)
+                       error = pm->restore(dev);
+       } else {
+               pci_pm_reenable_device(pci_dev);
+       }
 
        return error;
 }
 
-#else /* !CONFIG_HIBERNATION */
+#else /* !CONFIG_HIBERNATE_CALLBACKS */
 
 #define pci_pm_freeze          NULL
 #define pci_pm_freeze_noirq    NULL
@@ -748,9 +981,87 @@ static int pci_pm_restore(struct device *dev)
 #define pci_pm_restore         NULL
 #define pci_pm_restore_noirq   NULL
 
-#endif /* !CONFIG_HIBERNATION */
+#endif /* !CONFIG_HIBERNATE_CALLBACKS */
+
+#ifdef CONFIG_PM_RUNTIME
+
+static int pci_pm_runtime_suspend(struct device *dev)
+{
+       struct pci_dev *pci_dev = to_pci_dev(dev);
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
+       pci_power_t prev = pci_dev->current_state;
+       int error;
+
+       if (!pm || !pm->runtime_suspend)
+               return -ENOSYS;
+
+       error = pm->runtime_suspend(dev);
+       suspend_report_result(pm->runtime_suspend, error);
+       if (error)
+               return error;
+
+       pci_fixup_device(pci_fixup_suspend, pci_dev);
+
+       if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
+           && pci_dev->current_state != PCI_UNKNOWN) {
+               WARN_ONCE(pci_dev->current_state != prev,
+                       "PCI PM: State of device not saved by %pF\n",
+                       pm->runtime_suspend);
+               return 0;
+       }
+
+       if (!pci_dev->state_saved)
+               pci_save_state(pci_dev);
+
+       pci_finish_runtime_suspend(pci_dev);
+
+       return 0;
+}
+
+static int pci_pm_runtime_resume(struct device *dev)
+{
+       struct pci_dev *pci_dev = to_pci_dev(dev);
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
+
+       if (!pm || !pm->runtime_resume)
+               return -ENOSYS;
+
+       pci_pm_default_resume_early(pci_dev);
+       __pci_enable_wake(pci_dev, PCI_D0, true, false);
+       pci_fixup_device(pci_fixup_resume, pci_dev);
+
+       return pm->runtime_resume(dev);
+}
+
+static int pci_pm_runtime_idle(struct device *dev)
+{
+       const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
+
+       if (!pm)
+               return -ENOSYS;
 
-struct dev_pm_ops pci_dev_pm_ops = {
+       if (pm->runtime_idle) {
+               int ret = pm->runtime_idle(dev);
+               if (ret)
+                       return ret;
+       }
+
+       pm_runtime_suspend(dev);
+
+       return 0;
+}
+
+#else /* !CONFIG_PM_RUNTIME */
+
+#define pci_pm_runtime_suspend NULL
+#define pci_pm_runtime_resume  NULL
+#define pci_pm_runtime_idle    NULL
+
+#endif /* !CONFIG_PM_RUNTIME */
+
+#ifdef CONFIG_PM
+
+const struct dev_pm_ops pci_dev_pm_ops = {
        .prepare = pci_pm_prepare,
        .complete = pci_pm_complete,
        .suspend = pci_pm_suspend,
@@ -765,15 +1076,18 @@ struct dev_pm_ops pci_dev_pm_ops = {
        .thaw_noirq = pci_pm_thaw_noirq,
        .poweroff_noirq = pci_pm_poweroff_noirq,
        .restore_noirq = pci_pm_restore_noirq,
+       .runtime_suspend = pci_pm_runtime_suspend,
+       .runtime_resume = pci_pm_runtime_resume,
+       .runtime_idle = pci_pm_runtime_idle,
 };
 
 #define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
 
-#else /* !CONFIG_PM_SLEEP */
+#else /* !COMFIG_PM_OPS */
 
 #define PCI_PM_OPS_PTR NULL
 
-#endif /* !CONFIG_PM_SLEEP */
+#endif /* !COMFIG_PM_OPS */
 
 /**
  * __pci_register_driver - register a new pci driver
@@ -803,13 +1117,23 @@ int __pci_register_driver(struct pci_driver *drv, struct module *owner,
        /* register with core */
        error = driver_register(&drv->driver);
        if (error)
-               return error;
+               goto out;
 
        error = pci_create_newid_file(drv);
        if (error)
-               driver_unregister(&drv->driver);
+               goto out_newid;
 
+       error = pci_create_removeid_file(drv);
+       if (error)
+               goto out_removeid;
+out:
        return error;
+
+out_removeid:
+       pci_remove_newid_file(drv);
+out_newid:
+       driver_unregister(&drv->driver);
+       goto out;
 }
 
 /**
@@ -825,6 +1149,7 @@ int __pci_register_driver(struct pci_driver *drv, struct module *owner,
 void
 pci_unregister_driver(struct pci_driver *drv)
 {
+       pci_remove_removeid_file(drv);
        pci_remove_newid_file(drv);
        driver_unregister(&drv->driver);
        pci_free_dynids(drv);
@@ -924,6 +1249,7 @@ struct bus_type pci_bus_type = {
        .remove         = pci_device_remove,
        .shutdown       = pci_device_shutdown,
        .dev_attrs      = pci_dev_attrs,
+       .bus_attrs      = pci_bus_attrs,
        .pm             = PCI_PM_OPS_PTR,
 };
 
@@ -934,6 +1260,7 @@ static int __init pci_driver_init(void)
 
 postcore_initcall(pci_driver_init);
 
+EXPORT_SYMBOL_GPL(pci_add_dynid);
 EXPORT_SYMBOL(pci_match_id);
 EXPORT_SYMBOL(__pci_register_driver);
 EXPORT_SYMBOL(pci_unregister_driver);