Increment operation counts to track downloads.
[android/platform/packages/providers/DownloadProvider.git] / src / com / android / providers / downloads / DownloadThread.java
index e7763e4..28bbf49 100644 (file)
 
 package com.android.providers.downloads;
 
-import android.content.ContentUris;
+import static android.provider.Downloads.Impl.STATUS_BAD_REQUEST;
+import static android.provider.Downloads.Impl.STATUS_CANNOT_RESUME;
+import static android.provider.Downloads.Impl.STATUS_FILE_ERROR;
+import static android.provider.Downloads.Impl.STATUS_HTTP_DATA_ERROR;
+import static android.provider.Downloads.Impl.STATUS_SUCCESS;
+import static android.provider.Downloads.Impl.STATUS_TOO_MANY_REDIRECTS;
+import static android.provider.Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
+import static android.provider.Downloads.Impl.STATUS_WAITING_TO_RETRY;
+import static android.text.format.DateUtils.SECOND_IN_MILLIS;
+import static com.android.providers.downloads.Constants.TAG;
+import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
+import static java.net.HttpURLConnection.HTTP_MOVED_PERM;
+import static java.net.HttpURLConnection.HTTP_MOVED_TEMP;
+import static java.net.HttpURLConnection.HTTP_OK;
+import static java.net.HttpURLConnection.HTTP_PARTIAL;
+import static java.net.HttpURLConnection.HTTP_SEE_OTHER;
+import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+
 import android.content.ContentValues;
 import android.content.Context;
 import android.content.Intent;
-import android.drm.mobile1.DrmRawContent;
-import android.net.Uri;
-import android.net.http.AndroidHttpClient;
+import android.drm.DrmManagerClient;
+import android.drm.DrmOutputStream;
+import android.net.ConnectivityManager;
+import android.net.INetworkPolicyListener;
+import android.net.NetworkInfo;
+import android.net.NetworkPolicyManager;
+import android.net.TrafficStats;
 import android.os.FileUtils;
 import android.os.PowerManager;
 import android.os.Process;
+import android.os.SystemClock;
 import android.provider.Downloads;
-import android.provider.DrmStore;
-import android.util.Config;
+import android.text.TextUtils;
 import android.util.Log;
+import android.util.Pair;
+
+import com.android.providers.downloads.DownloadInfo.NetworkState;
 
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpGet;
+import libcore.io.IoUtils;
 
 import java.io.File;
-import java.io.FileNotFoundException;
+import java.io.FileDescriptor;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.SyncFailedException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Locale;
-import java.util.Map;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
 
 /**
- * Runs an actual download
+ * Task which executes a given {@link DownloadInfo}: making network requests,
+ * persisting data to disk, and updating {@link DownloadProvider}.
  */
-public class DownloadThread extends Thread {
+public class DownloadThread implements Runnable {
+
+    // TODO: bind each download to a specific network interface to avoid state
+    // checking races once we have ConnectivityManager API
+
+    private static final int HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416;
+    private static final int HTTP_TEMP_REDIRECT = 307;
+
+    private static final int DEFAULT_TIMEOUT = (int) (20 * SECOND_IN_MILLIS);
 
-    private Context mContext;
-    private DownloadInfo mInfo;
-    private SystemFacade mSystemFacade;
+    private final Context mContext;
+    private final DownloadInfo mInfo;
+    private final SystemFacade mSystemFacade;
+    private final StorageManager mStorageManager;
+    private final DownloadNotifier mNotifier;
 
-    public DownloadThread(Context context, SystemFacade systemFacade, DownloadInfo info) {
+    private volatile boolean mPolicyDirty;
+
+    public DownloadThread(Context context, SystemFacade systemFacade, DownloadInfo info,
+            StorageManager storageManager, DownloadNotifier notifier) {
         mContext = context;
         mSystemFacade = systemFacade;
         mInfo = info;
+        mStorageManager = storageManager;
+        mNotifier = notifier;
     }
 
     /**
@@ -66,8 +105,6 @@ public class DownloadThread extends Thread {
      */
     private String userAgent() {
         String userAgent = mInfo.mUserAgent;
-        if (userAgent != null) {
-        }
         if (userAgent == null) {
             userAgent = Constants.DEFAULT_USER_AGENT;
         }
@@ -75,657 +112,726 @@ public class DownloadThread extends Thread {
     }
 
     /**
-     * Executes the download in a separate thread
+     * State for the entire run() method.
      */
+    static class State {
+        public String mFilename;
+        public String mMimeType;
+        public int mRetryAfter = 0;
+        public boolean mGotData = false;
+        public String mRequestUri;
+        public long mTotalBytes = -1;
+        public long mCurrentBytes = 0;
+        public String mHeaderETag;
+        public boolean mContinuingDownload = false;
+        public long mBytesNotified = 0;
+        public long mTimeLastNotification = 0;
+        public int mNetworkType = ConnectivityManager.TYPE_NONE;
+
+        /** Historical bytes/second speed of this download. */
+        public long mSpeed;
+        /** Time when current sample started. */
+        public long mSpeedSampleStart;
+        /** Bytes transferred since current sample started. */
+        public long mSpeedSampleBytes;
+
+        public long mContentLength = -1;
+        public String mContentDisposition;
+        public String mContentLocation;
+
+        public int mRedirectionCount;
+        public URL mUrl;
+
+        public State(DownloadInfo info) {
+            mMimeType = Intent.normalizeMimeType(info.mMimeType);
+            mRequestUri = info.mUri;
+            mFilename = info.mFileName;
+            mTotalBytes = info.mTotalBytes;
+            mCurrentBytes = info.mCurrentBytes;
+        }
+
+        public void resetBeforeExecute() {
+            // Reset any state from previous execution
+            mContentLength = -1;
+            mContentDisposition = null;
+            mContentLocation = null;
+            mRedirectionCount = 0;
+        }
+    }
+
+    @Override
     public void run() {
         Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND);
+        try {
+            runInternal();
+        } finally {
+            mNotifier.notifyDownloadSpeed(mInfo.mId, 0);
+        }
+    }
 
-        int finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
-        boolean countRetry = false;
-        int retryAfter = 0;
-        int redirectCount = mInfo.mRedirectCount;
-        String newUri = null;
-        boolean gotData = false;
-        String filename = null;
-        String mimeType = sanitizeMimeType(mInfo.mMimeType);
-        FileOutputStream stream = null;
-        AndroidHttpClient client = null;
+    private void runInternal() {
+        // Skip when download already marked as finished; this download was
+        // probably started again while racing with UpdateThread.
+        if (DownloadInfo.queryDownloadStatus(mContext.getContentResolver(), mInfo.mId)
+                == Downloads.Impl.STATUS_SUCCESS) {
+            Log.d(TAG, "Download " + mInfo.mId + " already finished; skipping");
+            return;
+        }
+
+        State state = new State(mInfo);
         PowerManager.WakeLock wakeLock = null;
-        Uri contentUri = Uri.parse(Downloads.Impl.CONTENT_URI + "/" + mInfo.mId);
+        int finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
+        int numFailed = mInfo.mNumFailed;
+        String errorMsg = null;
+
+        final NetworkPolicyManager netPolicy = NetworkPolicyManager.from(mContext);
+        final PowerManager pm = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE);
 
         try {
-            boolean continuingDownload = false;
-            String headerAcceptRanges = null;
-            String headerContentDisposition = null;
-            String headerContentLength = null;
-            String headerContentLocation = null;
-            String headerETag = null;
-            String headerTransferEncoding = null;
+            wakeLock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, Constants.TAG);
+            wakeLock.acquire();
 
-            byte data[] = new byte[Constants.BUFFER_SIZE];
+            // while performing download, register for rules updates
+            netPolicy.registerListener(mPolicyListener);
 
-            int bytesSoFar = 0;
+            Log.i(Constants.TAG, "Download " + mInfo.mId + " starting");
 
-            PowerManager pm = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE);
-            wakeLock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, Constants.TAG);
-            wakeLock.acquire();
+            // Remember which network this download started on; used to
+            // determine if errors were due to network changes.
+            final NetworkInfo info = mSystemFacade.getActiveNetworkInfo(mInfo.mUid);
+            if (info != null) {
+                state.mNetworkType = info.getType();
+            }
+
+            // Network traffic on this thread should be counted against the
+            // requesting UID, and is tagged with well-known value.
+            TrafficStats.setThreadStatsTag(TrafficStats.TAG_SYSTEM_DOWNLOAD);
+            TrafficStats.setThreadStatsUid(mInfo.mUid);
+
+            try {
+                // TODO: migrate URL sanity checking into client side of API
+                state.mUrl = new URL(state.mRequestUri);
+            } catch (MalformedURLException e) {
+                throw new StopRequestException(STATUS_BAD_REQUEST, e);
+            }
+
+            executeDownload(state);
 
-            filename = mInfo.mFileName;
-            if (filename != null) {
-                if (!Helpers.isFilenameValid(filename)) {
-                    finalStatus = Downloads.Impl.STATUS_FILE_ERROR;
-                    notifyDownloadCompleted(
-                            finalStatus, false, 0, 0, false, filename, null, mInfo.mMimeType);
-                    return;
+            finalizeDestinationFile(state);
+            finalStatus = Downloads.Impl.STATUS_SUCCESS;
+        } catch (StopRequestException error) {
+            // remove the cause before printing, in case it contains PII
+            errorMsg = error.getMessage();
+            String msg = "Aborting request for download " + mInfo.mId + ": " + errorMsg;
+            Log.w(Constants.TAG, msg);
+            if (Constants.LOGV) {
+                Log.w(Constants.TAG, msg, error);
+            }
+            finalStatus = error.getFinalStatus();
+
+            // Nobody below our level should request retries, since we handle
+            // failure counts at this level.
+            if (finalStatus == STATUS_WAITING_TO_RETRY) {
+                throw new IllegalStateException("Execution should always throw final error codes");
+            }
+
+            // Some errors should be retryable, unless we fail too many times.
+            if (isStatusRetryable(finalStatus)) {
+                if (state.mGotData) {
+                    numFailed = 1;
+                } else {
+                    numFailed += 1;
                 }
-                // We're resuming a download that got interrupted
-                File f = new File(filename);
-                if (f.exists()) {
-                    long fileLength = f.length();
-                    if (fileLength == 0) {
-                        // The download hadn't actually started, we can restart from scratch
-                        f.delete();
-                        filename = null;
-                    } else if (mInfo.mETag == null && !mInfo.mNoIntegrity) {
-                        // Tough luck, that's not a resumable download
-                        if (Config.LOGD) {
-                            Log.d(Constants.TAG,
-                                    "can't resume interrupted non-resumable download");
-                        }
-                        f.delete();
-                        finalStatus = Downloads.Impl.STATUS_PRECONDITION_FAILED;
-                        notifyDownloadCompleted(
-                                finalStatus, false, 0, 0, false, filename, null, mInfo.mMimeType);
-                        return;
+
+                if (numFailed < Constants.MAX_RETRIES) {
+                    final NetworkInfo info = mSystemFacade.getActiveNetworkInfo(mInfo.mUid);
+                    if (info != null && info.getType() == state.mNetworkType
+                            && info.isConnected()) {
+                        // Underlying network is still intact, use normal backoff
+                        finalStatus = STATUS_WAITING_TO_RETRY;
                     } else {
-                        // All right, we'll be able to resume this download
-                        stream = new FileOutputStream(filename, true);
-                        bytesSoFar = (int) fileLength;
-                        if (mInfo.mTotalBytes != -1) {
-                            headerContentLength = Integer.toString(mInfo.mTotalBytes);
-                        }
-                        headerETag = mInfo.mETag;
-                        continuingDownload = true;
+                        // Network changed, retry on any next available
+                        finalStatus = STATUS_WAITING_FOR_NETWORK;
                     }
                 }
             }
 
-            int bytesNotified = bytesSoFar;
-            // starting with MIN_VALUE means that the first write will commit
-            //     progress to the database
-            long timeLastNotification = 0;
+            // fall through to finally block
+        } catch (Throwable ex) {
+            errorMsg = ex.getMessage();
+            String msg = "Exception for id " + mInfo.mId + ": " + errorMsg;
+            Log.w(Constants.TAG, msg, ex);
+            finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
+            // falls through to the code that reports an error
+        } finally {
+            if (finalStatus == STATUS_SUCCESS) {
+                TrafficStats.incrementOperationCount(1);
+            }
 
-            client = AndroidHttpClient.newInstance(userAgent(), mContext);
+            TrafficStats.clearThreadStatsTag();
+            TrafficStats.clearThreadStatsUid();
 
-            if (stream != null && mInfo.mDestination == Downloads.Impl.DESTINATION_EXTERNAL
-                        && !DrmRawContent.DRM_MIMETYPE_MESSAGE_STRING
-                        .equalsIgnoreCase(mimeType)) {
-                try {
-                    stream.close();
-                    stream = null;
-                } catch (IOException ex) {
-                    if (Constants.LOGV) {
-                        Log.v(Constants.TAG, "exception when closing the file before download : " +
-                                ex);
-                    }
-                    // nothing can really be done if the file can't be closed
-                }
-            }
+            cleanupDestination(state, finalStatus);
+            notifyDownloadCompleted(state, finalStatus, errorMsg, numFailed);
 
-            /*
-             * This loop is run once for every individual HTTP request that gets sent.
-             * The very first HTTP request is a "virgin" request, while every subsequent
-             * request is done with the original ETag and a byte-range.
-             */
-http_request_loop:
-            while (true) {
-                // Prepares the request and fires it.
-                HttpGet request = new HttpGet(mInfo.mUri);
+            Log.i(Constants.TAG, "Download " + mInfo.mId + " finished with status "
+                    + Downloads.Impl.statusToString(finalStatus));
 
-                if (Constants.LOGV) {
-                    Log.v(Constants.TAG, "initiating download for " + mInfo.mUri);
-                }
+            netPolicy.unregisterListener(mPolicyListener);
 
-                addRequestHeaders(request);
+            if (wakeLock != null) {
+                wakeLock.release();
+                wakeLock = null;
+            }
+        }
+        mStorageManager.incrementNumDownloadsSoFar();
+    }
 
-                if (continuingDownload) {
-                    if (headerETag != null) {
-                        request.addHeader("If-Match", headerETag);
-                    }
-                    request.addHeader("Range", "bytes=" + bytesSoFar + "-");
-                }
+    /**
+     * Fully execute a single download request. Setup and send the request,
+     * handle the response, and transfer the data to the destination file.
+     */
+    private void executeDownload(State state) throws StopRequestException {
+        state.resetBeforeExecute();
+        setupDestinationFile(state);
+
+        // skip when already finished; remove after fixing race in 5217390
+        if (state.mCurrentBytes == state.mTotalBytes) {
+            Log.i(Constants.TAG, "Skipping initiating request for download " +
+                  mInfo.mId + "; already completed");
+            return;
+        }
 
-                HttpResponse response;
-                try {
-                    response = client.execute(request);
-                } catch (IllegalArgumentException ex) {
-                    if (Constants.LOGV) {
-                        Log.d(Constants.TAG, "Arg exception trying to execute request for " +
-                                mInfo.mUri + " : " + ex);
-                    } else if (Config.LOGD) {
-                        Log.d(Constants.TAG, "Arg exception trying to execute request for " +
-                                mInfo.mId + " : " +  ex);
-                    }
-                    finalStatus = Downloads.Impl.STATUS_BAD_REQUEST;
-                    request.abort();
-                    break http_request_loop;
-                } catch (IOException ex) {
-                    if (Constants.LOGX) {
-                        if (Helpers.isNetworkAvailable(mSystemFacade)) {
-                            Log.i(Constants.TAG, "Execute Failed " + mInfo.mId + ", Net Up");
-                        } else {
-                            Log.i(Constants.TAG, "Execute Failed " + mInfo.mId + ", Net Down");
-                        }
-                    }
-                    if (!Helpers.isNetworkAvailable(mSystemFacade)) {
-                        finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                    } else if (mInfo.mNumFailed < Constants.MAX_RETRIES) {
-                        finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                        countRetry = true;
-                    } else {
-                        if (Constants.LOGV) {
-                            Log.d(Constants.TAG, "IOException trying to execute request for " +
-                                    mInfo.mUri + " : " + ex);
-                        } else if (Config.LOGD) {
-                            Log.d(Constants.TAG, "IOException trying to execute request for " +
-                                    mInfo.mId + " : " + ex);
+        while (state.mRedirectionCount++ < Constants.MAX_REDIRECTS) {
+            // Open connection and follow any redirects until we have a useful
+            // response with body.
+            HttpURLConnection conn = null;
+            try {
+                checkConnectivity();
+                conn = (HttpURLConnection) state.mUrl.openConnection();
+                conn.setInstanceFollowRedirects(false);
+                conn.setConnectTimeout(DEFAULT_TIMEOUT);
+                conn.setReadTimeout(DEFAULT_TIMEOUT);
+
+                addRequestHeaders(state, conn);
+
+                final int responseCode = conn.getResponseCode();
+                switch (responseCode) {
+                    case HTTP_OK:
+                        if (state.mContinuingDownload) {
+                            throw new StopRequestException(
+                                    STATUS_CANNOT_RESUME, "Expected partial, but received OK");
                         }
-                        finalStatus = Downloads.Impl.STATUS_HTTP_DATA_ERROR;
-                    }
-                    request.abort();
-                    break http_request_loop;
-                }
+                        processResponseHeaders(state, conn);
+                        transferData(state, conn);
+                        return;
 
-                int statusCode = response.getStatusLine().getStatusCode();
-                if (statusCode == 503 && mInfo.mNumFailed < Constants.MAX_RETRIES) {
-                    if (Constants.LOGVV) {
-                        Log.v(Constants.TAG, "got HTTP response code 503");
-                    }
-                    finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                    countRetry = true;
-                    Header header = response.getFirstHeader("Retry-After");
-                    if (header != null) {
-                       try {
-                           if (Constants.LOGVV) {
-                               Log.v(Constants.TAG, "Retry-After :" + header.getValue());
-                           }
-                           retryAfter = Integer.parseInt(header.getValue());
-                           if (retryAfter < 0) {
-                               retryAfter = 0;
-                           } else {
-                               if (retryAfter < Constants.MIN_RETRY_AFTER) {
-                                   retryAfter = Constants.MIN_RETRY_AFTER;
-                               } else if (retryAfter > Constants.MAX_RETRY_AFTER) {
-                                   retryAfter = Constants.MAX_RETRY_AFTER;
-                               }
-                               retryAfter += Helpers.sRandom.nextInt(Constants.MIN_RETRY_AFTER + 1);
-                               retryAfter *= 1000;
-                           }
-                       } catch (NumberFormatException ex) {
-                           // ignored - retryAfter stays 0 in this case.
-                       }
-                    }
-                    request.abort();
-                    break http_request_loop;
-                }
-                if (statusCode == 301 ||
-                        statusCode == 302 ||
-                        statusCode == 303 ||
-                        statusCode == 307) {
-                    if (Constants.LOGVV) {
-                        Log.v(Constants.TAG, "got HTTP redirect " + statusCode);
-                    }
-                    if (redirectCount >= Constants.MAX_REDIRECTS) {
-                        if (Constants.LOGV) {
-                            Log.d(Constants.TAG, "too many redirects for download " + mInfo.mId +
-                                    " at " + mInfo.mUri);
-                        } else if (Config.LOGD) {
-                            Log.d(Constants.TAG, "too many redirects for download " + mInfo.mId);
-                        }
-                        finalStatus = Downloads.Impl.STATUS_TOO_MANY_REDIRECTS;
-                        request.abort();
-                        break http_request_loop;
-                    }
-                    Header header = response.getFirstHeader("Location");
-                    if (header != null) {
-                        if (Constants.LOGVV) {
-                            Log.v(Constants.TAG, "Location :" + header.getValue());
+                    case HTTP_PARTIAL:
+                        if (!state.mContinuingDownload) {
+                            throw new StopRequestException(
+                                    STATUS_CANNOT_RESUME, "Expected OK, but received partial");
                         }
-                        try {
-                            newUri = new URI(mInfo.mUri).
-                                    resolve(new URI(header.getValue())).
-                                    toString();
-                        } catch(URISyntaxException ex) {
-                            if (Constants.LOGV) {
-                                Log.d(Constants.TAG,
-                                        "Couldn't resolve redirect URI " +
-                                        header.getValue() +
-                                        " for " +
-                                        mInfo.mUri);
-                            } else if (Config.LOGD) {
-                                Log.d(Constants.TAG,
-                                        "Couldn't resolve redirect URI for download " +
-                                        mInfo.mId);
-                            }
-                            finalStatus = Downloads.Impl.STATUS_BAD_REQUEST;
-                            request.abort();
-                            break http_request_loop;
+                        transferData(state, conn);
+                        return;
+
+                    case HTTP_MOVED_PERM:
+                    case HTTP_MOVED_TEMP:
+                    case HTTP_SEE_OTHER:
+                    case HTTP_TEMP_REDIRECT:
+                        final String location = conn.getHeaderField("Location");
+                        state.mUrl = new URL(state.mUrl, location);
+                        if (responseCode == HTTP_MOVED_PERM) {
+                            // Push updated URL back to database
+                            state.mRequestUri = state.mUrl.toString();
                         }
-                        ++redirectCount;
-                        finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                        request.abort();
-                        break http_request_loop;
-                    }
+                        continue;
+
+                    case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
+                        throw new StopRequestException(
+                                STATUS_CANNOT_RESUME, "Requested range not satisfiable");
+
+                    case HTTP_UNAVAILABLE:
+                        parseRetryAfterHeaders(state, conn);
+                        throw new StopRequestException(
+                                HTTP_UNAVAILABLE, conn.getResponseMessage());
+
+                    case HTTP_INTERNAL_ERROR:
+                        throw new StopRequestException(
+                                HTTP_INTERNAL_ERROR, conn.getResponseMessage());
+
+                    default:
+                        StopRequestException.throwUnhandledHttpError(
+                                responseCode, conn.getResponseMessage());
                 }
-                if ((!continuingDownload && statusCode != Downloads.Impl.STATUS_SUCCESS)
-                        || (continuingDownload && statusCode != 206)) {
-                    if (Constants.LOGV) {
-                        Log.d(Constants.TAG, "http error " + statusCode + " for " + mInfo.mUri);
-                    } else if (Config.LOGD) {
-                        Log.d(Constants.TAG, "http error " + statusCode + " for download " +
-                                mInfo.mId);
-                    }
-                    if (Downloads.Impl.isStatusError(statusCode)) {
-                        finalStatus = statusCode;
-                    } else if (statusCode >= 300 && statusCode < 400) {
-                        finalStatus = Downloads.Impl.STATUS_UNHANDLED_REDIRECT;
-                    } else if (continuingDownload && statusCode == Downloads.Impl.STATUS_SUCCESS) {
-                        finalStatus = Downloads.Impl.STATUS_PRECONDITION_FAILED;
-                    } else {
-                        finalStatus = Downloads.Impl.STATUS_UNHANDLED_HTTP_CODE;
-                    }
-                    request.abort();
-                    break http_request_loop;
-                } else {
-                    // Handles the response, saves the file
-                    if (Constants.LOGV) {
-                        Log.v(Constants.TAG, "received response for " + mInfo.mUri);
-                    }
+            } catch (IOException e) {
+                // Trouble with low-level sockets
+                throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
 
-                    if (!continuingDownload) {
-                        Header header = response.getFirstHeader("Accept-Ranges");
-                        if (header != null) {
-                            headerAcceptRanges = header.getValue();
-                        }
-                        header = response.getFirstHeader("Content-Disposition");
-                        if (header != null) {
-                            headerContentDisposition = header.getValue();
-                        }
-                        header = response.getFirstHeader("Content-Location");
-                        if (header != null) {
-                            headerContentLocation = header.getValue();
-                        }
-                        if (mimeType == null) {
-                            header = response.getFirstHeader("Content-Type");
-                            if (header != null) {
-                                mimeType = sanitizeMimeType(header.getValue());
-                            }
-                        }
-                        header = response.getFirstHeader("ETag");
-                        if (header != null) {
-                            headerETag = header.getValue();
-                        }
-                        header = response.getFirstHeader("Transfer-Encoding");
-                        if (header != null) {
-                            headerTransferEncoding = header.getValue();
-                        }
-                        if (headerTransferEncoding == null) {
-                            header = response.getFirstHeader("Content-Length");
-                            if (header != null) {
-                                headerContentLength = header.getValue();
-                            }
-                        } else {
-                            // Ignore content-length with transfer-encoding - 2616 4.4 3
-                            if (Constants.LOGVV) {
-                                Log.v(Constants.TAG,
-                                        "ignoring content-length because of xfer-encoding");
-                            }
-                        }
-                        if (Constants.LOGVV) {
-                            Log.v(Constants.TAG, "Accept-Ranges: " + headerAcceptRanges);
-                            Log.v(Constants.TAG, "Content-Disposition: " +
-                                    headerContentDisposition);
-                            Log.v(Constants.TAG, "Content-Length: " + headerContentLength);
-                            Log.v(Constants.TAG, "Content-Location: " + headerContentLocation);
-                            Log.v(Constants.TAG, "Content-Type: " + mimeType);
-                            Log.v(Constants.TAG, "ETag: " + headerETag);
-                            Log.v(Constants.TAG, "Transfer-Encoding: " + headerTransferEncoding);
-                        }
+            } finally {
+                if (conn != null) conn.disconnect();
+            }
+        }
 
-                        if (!mInfo.mNoIntegrity && headerContentLength == null &&
-                                (headerTransferEncoding == null
-                                        || !headerTransferEncoding.equalsIgnoreCase("chunked"))
-                                ) {
-                            if (Config.LOGD) {
-                                Log.d(Constants.TAG, "can't know size of download, giving up");
-                            }
-                            finalStatus = Downloads.Impl.STATUS_LENGTH_REQUIRED;
-                            request.abort();
-                            break http_request_loop;
-                        }
+        throw new StopRequestException(STATUS_TOO_MANY_REDIRECTS, "Too many redirects");
+    }
 
-                        DownloadFileInfo fileInfo = Helpers.generateSaveFile(
-                                mContext,
-                                mInfo.mUri,
-                                mInfo.mHint,
-                                headerContentDisposition,
-                                headerContentLocation,
-                                mimeType,
-                                mInfo.mDestination,
-                                (headerContentLength != null) ?
-                                        Integer.parseInt(headerContentLength) : 0);
-                        if (fileInfo.mFileName == null) {
-                            finalStatus = fileInfo.mStatus;
-                            request.abort();
-                            break http_request_loop;
-                        }
-                        filename = fileInfo.mFileName;
-                        stream = fileInfo.mStream;
-                        if (Constants.LOGV) {
-                            Log.v(Constants.TAG, "writing " + mInfo.mUri + " to " + filename);
-                        }
+    /**
+     * Transfer data from the given connection to the destination file.
+     */
+    private void transferData(State state, HttpURLConnection conn) throws StopRequestException {
+        DrmManagerClient drmClient = null;
+        InputStream in = null;
+        OutputStream out = null;
+        FileDescriptor outFd = null;
+        try {
+            try {
+                in = conn.getInputStream();
+            } catch (IOException e) {
+                throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
+            }
 
-                        ContentValues values = new ContentValues();
-                        values.put(Downloads.Impl._DATA, filename);
-                        if (headerETag != null) {
-                            values.put(Constants.ETAG, headerETag);
-                        }
-                        if (mimeType != null) {
-                            values.put(Downloads.Impl.COLUMN_MIME_TYPE, mimeType);
-                        }
-                        int contentLength = -1;
-                        if (headerContentLength != null) {
-                            contentLength = Integer.parseInt(headerContentLength);
-                        }
-                        values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, contentLength);
-                        mContext.getContentResolver().update(contentUri, values, null, null);
-                    }
+            try {
+                if (DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType)) {
+                    drmClient = new DrmManagerClient(mContext);
+                    final RandomAccessFile file = new RandomAccessFile(
+                            new File(state.mFilename), "rw");
+                    out = new DrmOutputStream(drmClient, file, state.mMimeType);
+                    outFd = file.getFD();
+                } else {
+                    out = new FileOutputStream(state.mFilename, true);
+                    outFd = ((FileOutputStream) out).getFD();
+                }
+            } catch (IOException e) {
+                throw new StopRequestException(STATUS_FILE_ERROR, e);
+            }
 
-                    InputStream entityStream;
-                    try {
-                        entityStream = response.getEntity().getContent();
-                    } catch (IOException ex) {
-                        if (Constants.LOGX) {
-                            if (Helpers.isNetworkAvailable(mSystemFacade)) {
-                                Log.i(Constants.TAG, "Get Failed " + mInfo.mId + ", Net Up");
-                            } else {
-                                Log.i(Constants.TAG, "Get Failed " + mInfo.mId + ", Net Down");
-                            }
-                        }
-                        if (!Helpers.isNetworkAvailable(mSystemFacade)) {
-                            finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                        } else if (mInfo.mNumFailed < Constants.MAX_RETRIES) {
-                            finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                            countRetry = true;
-                        } else {
-                            if (Constants.LOGV) {
-                                Log.d(Constants.TAG,
-                                        "IOException getting entity for " +
-                                        mInfo.mUri +
-                                        " : " +
-                                        ex);
-                            } else if (Config.LOGD) {
-                                Log.d(Constants.TAG, "IOException getting entity for download " +
-                                        mInfo.mId + " : " + ex);
-                            }
-                            finalStatus = Downloads.Impl.STATUS_HTTP_DATA_ERROR;
-                        }
-                        request.abort();
-                        break http_request_loop;
-                    }
-                    for (;;) {
-                        int bytesRead;
-                        try {
-                            bytesRead = entityStream.read(data);
-                        } catch (IOException ex) {
-                            if (Constants.LOGX) {
-                                if (Helpers.isNetworkAvailable(mSystemFacade)) {
-                                    Log.i(Constants.TAG, "Read Failed " + mInfo.mId + ", Net Up");
-                                } else {
-                                    Log.i(Constants.TAG, "Read Failed " + mInfo.mId + ", Net Down");
-                                }
-                            }
-                            ContentValues values = new ContentValues();
-                            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, bytesSoFar);
-                            mContext.getContentResolver().update(contentUri, values, null, null);
-                            if (!mInfo.mNoIntegrity && headerETag == null) {
-                                if (Constants.LOGV) {
-                                    Log.v(Constants.TAG, "download IOException for " + mInfo.mUri +
-                                            " : " + ex);
-                                } else if (Config.LOGD) {
-                                    Log.d(Constants.TAG, "download IOException for download " +
-                                            mInfo.mId + " : " + ex);
-                                }
-                                if (Config.LOGD) {
-                                    Log.d(Constants.TAG,
-                                            "can't resume interrupted download with no ETag");
-                                }
-                                finalStatus = Downloads.Impl.STATUS_PRECONDITION_FAILED;
-                            } else if (!Helpers.isNetworkAvailable(mSystemFacade)) {
-                                finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                            } else if (mInfo.mNumFailed < Constants.MAX_RETRIES) {
-                                finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                                countRetry = true;
-                            } else {
-                                if (Constants.LOGV) {
-                                    Log.v(Constants.TAG, "download IOException for " + mInfo.mUri +
-                                            " : " + ex);
-                                } else if (Config.LOGD) {
-                                    Log.d(Constants.TAG, "download IOException for download " +
-                                            mInfo.mId + " : " + ex);
-                                }
-                                finalStatus = Downloads.Impl.STATUS_HTTP_DATA_ERROR;
-                            }
-                            request.abort();
-                            break http_request_loop;
-                        }
-                        if (bytesRead == -1) { // success
-                            ContentValues values = new ContentValues();
-                            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, bytesSoFar);
-                            if (headerContentLength == null) {
-                                values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, bytesSoFar);
-                            }
-                            mContext.getContentResolver().update(contentUri, values, null, null);
-                            if ((headerContentLength != null)
-                                    && (bytesSoFar
-                                            != Integer.parseInt(headerContentLength))) {
-                                if (!mInfo.mNoIntegrity && headerETag == null) {
-                                    if (Constants.LOGV) {
-                                        Log.d(Constants.TAG, "mismatched content length " +
-                                                mInfo.mUri);
-                                    } else if (Config.LOGD) {
-                                        Log.d(Constants.TAG, "mismatched content length for " +
-                                                mInfo.mId);
-                                    }
-                                    finalStatus = Downloads.Impl.STATUS_LENGTH_REQUIRED;
-                                } else if (!Helpers.isNetworkAvailable(mSystemFacade)) {
-                                    finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                                } else if (mInfo.mNumFailed < Constants.MAX_RETRIES) {
-                                    finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                                    countRetry = true;
-                                } else {
-                                    if (Constants.LOGV) {
-                                        Log.v(Constants.TAG, "closed socket for " + mInfo.mUri);
-                                    } else if (Config.LOGD) {
-                                        Log.d(Constants.TAG, "closed socket for download " +
-                                                mInfo.mId);
-                                    }
-                                    finalStatus = Downloads.Impl.STATUS_HTTP_DATA_ERROR;
-                                }
-                                break http_request_loop;
-                            }
-                            break;
-                        }
-                        gotData = true;
-                        for (;;) {
-                            try {
-                                if (stream == null) {
-                                    stream = new FileOutputStream(filename, true);
-                                }
-                                stream.write(data, 0, bytesRead);
-                                if (mInfo.mDestination == Downloads.Impl.DESTINATION_EXTERNAL
-                                            && !DrmRawContent.DRM_MIMETYPE_MESSAGE_STRING
-                                            .equalsIgnoreCase(mimeType)) {
-                                    try {
-                                        stream.close();
-                                        stream = null;
-                                    } catch (IOException ex) {
-                                        if (Constants.LOGV) {
-                                            Log.v(Constants.TAG,
-                                                    "exception when closing the file " +
-                                                    "during download : " + ex);
-                                        }
-                                        // nothing can really be done if the file can't be closed
-                                    }
-                                }
-                                break;
-                            } catch (IOException ex) {
-                                if (!Helpers.discardPurgeableFiles(
-                                        mContext, Constants.BUFFER_SIZE)) {
-                                    finalStatus = Downloads.Impl.STATUS_FILE_ERROR;
-                                    break http_request_loop;
-                                }
-                            }
-                        }
-                        bytesSoFar += bytesRead;
-                        long now = mSystemFacade.currentTimeMillis();
-                        if (bytesSoFar - bytesNotified > Constants.MIN_PROGRESS_STEP
-                                && now - timeLastNotification
-                                        > Constants.MIN_PROGRESS_TIME) {
-                            ContentValues values = new ContentValues();
-                            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, bytesSoFar);
-                            mContext.getContentResolver().update(
-                                    contentUri, values, null, null);
-                            bytesNotified = bytesSoFar;
-                            timeLastNotification = now;
-                        }
+            // Start streaming data, periodically watch for pause/cancel
+            // commands and checking disk space as needed.
+            transferData(state, in, out);
 
-                        if (Constants.LOGVV) {
-                            Log.v(Constants.TAG, "downloaded " + bytesSoFar + " for " + mInfo.mUri);
-                        }
-                        synchronized (mInfo) {
-                            if (mInfo.mControl == Downloads.Impl.CONTROL_PAUSED) {
-                                if (Constants.LOGV) {
-                                    Log.v(Constants.TAG, "paused " + mInfo.mUri);
-                                }
-                                finalStatus = Downloads.Impl.STATUS_RUNNING_PAUSED;
-                                request.abort();
-                                break http_request_loop;
-                            }
-                        }
-                        if (mInfo.mStatus == Downloads.Impl.STATUS_CANCELED) {
-                            if (Constants.LOGV) {
-                                Log.d(Constants.TAG, "canceled " + mInfo.mUri);
-                            } else if (Config.LOGD) {
-                                // Log.d(Constants.TAG, "canceled id " + mInfo.mId);
-                            }
-                            finalStatus = Downloads.Impl.STATUS_CANCELED;
-                            break http_request_loop;
-                        }
-                    }
-                    if (Constants.LOGV) {
-                        Log.v(Constants.TAG, "download completed for " + mInfo.mUri);
-                    }
-                    finalStatus = Downloads.Impl.STATUS_SUCCESS;
+            try {
+                if (out instanceof DrmOutputStream) {
+                    ((DrmOutputStream) out).finish();
                 }
-                break;
+            } catch (IOException e) {
+                throw new StopRequestException(STATUS_FILE_ERROR, e);
             }
-        } catch (FileNotFoundException ex) {
-            if (Config.LOGD) {
-                Log.d(Constants.TAG, "FileNotFoundException for " + filename + " : " +  ex);
+
+        } finally {
+            if (drmClient != null) {
+                drmClient.release();
             }
-            finalStatus = Downloads.Impl.STATUS_FILE_ERROR;
-            // falls through to the code that reports an error
-        } catch (RuntimeException ex) { //sometimes the socket code throws unchecked exceptions
-            if (Constants.LOGV) {
-                Log.d(Constants.TAG, "Exception for " + mInfo.mUri, ex);
-            } else if (Config.LOGD) {
-                Log.d(Constants.TAG, "Exception for id " + mInfo.mId, ex);
+
+            IoUtils.closeQuietly(in);
+
+            try {
+                if (out != null) out.flush();
+                if (outFd != null) outFd.sync();
+            } catch (IOException e) {
+            } finally {
+                IoUtils.closeQuietly(out);
             }
-            finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
-            // falls through to the code that reports an error
-        } finally {
-            mInfo.mHasActiveThread = false;
-            if (wakeLock != null) {
-                wakeLock.release();
-                wakeLock = null;
+        }
+    }
+
+    /**
+     * Check if current connectivity is valid for this request.
+     */
+    private void checkConnectivity() throws StopRequestException {
+        // checking connectivity will apply current policy
+        mPolicyDirty = false;
+
+        final NetworkState networkUsable = mInfo.checkCanUseNetwork();
+        if (networkUsable != NetworkState.OK) {
+            int status = Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
+            if (networkUsable == NetworkState.UNUSABLE_DUE_TO_SIZE) {
+                status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
+                mInfo.notifyPauseDueToSize(true);
+            } else if (networkUsable == NetworkState.RECOMMENDED_UNUSABLE_DUE_TO_SIZE) {
+                status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
+                mInfo.notifyPauseDueToSize(false);
+            }
+            throw new StopRequestException(status, networkUsable.name());
+        }
+    }
+
+    /**
+     * Transfer as much data as possible from the HTTP response to the
+     * destination file.
+     */
+    private void transferData(State state, InputStream in, OutputStream out)
+            throws StopRequestException {
+        final byte data[] = new byte[Constants.BUFFER_SIZE];
+        for (;;) {
+            int bytesRead = readFromResponse(state, data, in);
+            if (bytesRead == -1) { // success, end of stream already reached
+                handleEndOfStream(state);
+                return;
+            }
+
+            state.mGotData = true;
+            writeDataToDestination(state, data, bytesRead, out);
+            state.mCurrentBytes += bytesRead;
+            reportProgress(state);
+
+            if (Constants.LOGVV) {
+                Log.v(Constants.TAG, "downloaded " + state.mCurrentBytes + " for "
+                      + mInfo.mUri);
+            }
+
+            checkPausedOrCanceled(state);
+        }
+    }
+
+    /**
+     * Called after a successful completion to take any necessary action on the downloaded file.
+     */
+    private void finalizeDestinationFile(State state) {
+        if (state.mFilename != null) {
+            // make sure the file is readable
+            FileUtils.setPermissions(state.mFilename, 0644, -1, -1);
+        }
+    }
+
+    /**
+     * Called just before the thread finishes, regardless of status, to take any necessary action on
+     * the downloaded file.
+     */
+    private void cleanupDestination(State state, int finalStatus) {
+        if (state.mFilename != null && Downloads.Impl.isStatusError(finalStatus)) {
+            if (Constants.LOGVV) {
+                Log.d(TAG, "cleanupDestination() deleting " + state.mFilename);
+            }
+            new File(state.mFilename).delete();
+            state.mFilename = null;
+        }
+    }
+
+    /**
+     * Check if the download has been paused or canceled, stopping the request appropriately if it
+     * has been.
+     */
+    private void checkPausedOrCanceled(State state) throws StopRequestException {
+        synchronized (mInfo) {
+            if (mInfo.mControl == Downloads.Impl.CONTROL_PAUSED) {
+                throw new StopRequestException(
+                        Downloads.Impl.STATUS_PAUSED_BY_APP, "download paused by owner");
+            }
+            if (mInfo.mStatus == Downloads.Impl.STATUS_CANCELED) {
+                throw new StopRequestException(Downloads.Impl.STATUS_CANCELED, "download canceled");
+            }
+        }
+
+        // if policy has been changed, trigger connectivity check
+        if (mPolicyDirty) {
+            checkConnectivity();
+        }
+    }
+
+    /**
+     * Report download progress through the database if necessary.
+     */
+    private void reportProgress(State state) {
+        final long now = SystemClock.elapsedRealtime();
+
+        final long sampleDelta = now - state.mSpeedSampleStart;
+        if (sampleDelta > 500) {
+            final long sampleSpeed = ((state.mCurrentBytes - state.mSpeedSampleBytes) * 1000)
+                    / sampleDelta;
+
+            if (state.mSpeed == 0) {
+                state.mSpeed = sampleSpeed;
+            } else {
+                state.mSpeed = ((state.mSpeed * 3) + sampleSpeed) / 4;
             }
-            if (client != null) {
-                client.close();
-                client = null;
+
+            // Only notify once we have a full sample window
+            if (state.mSpeedSampleStart != 0) {
+                mNotifier.notifyDownloadSpeed(mInfo.mId, state.mSpeed);
             }
+
+            state.mSpeedSampleStart = now;
+            state.mSpeedSampleBytes = state.mCurrentBytes;
+        }
+
+        if (state.mCurrentBytes - state.mBytesNotified > Constants.MIN_PROGRESS_STEP &&
+            now - state.mTimeLastNotification > Constants.MIN_PROGRESS_TIME) {
+            ContentValues values = new ContentValues();
+            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
+            mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
+            state.mBytesNotified = state.mCurrentBytes;
+            state.mTimeLastNotification = now;
+        }
+    }
+
+    /**
+     * Write a data buffer to the destination file.
+     * @param data buffer containing the data to write
+     * @param bytesRead how many bytes to write from the buffer
+     */
+    private void writeDataToDestination(State state, byte[] data, int bytesRead, OutputStream out)
+            throws StopRequestException {
+        mStorageManager.verifySpaceBeforeWritingToFile(
+                mInfo.mDestination, state.mFilename, bytesRead);
+
+        boolean forceVerified = false;
+        while (true) {
             try {
-                // close the file
-                if (stream != null) {
-                    stream.close();
-                }
+                out.write(data, 0, bytesRead);
+                return;
             } catch (IOException ex) {
-                if (Constants.LOGV) {
-                    Log.v(Constants.TAG, "exception when closing the file after download : " + ex);
+                // TODO: better differentiate between DRM and disk failures
+                if (!forceVerified) {
+                    // couldn't write to file. are we out of space? check.
+                    mStorageManager.verifySpace(mInfo.mDestination, state.mFilename, bytesRead);
+                    forceVerified = true;
+                } else {
+                    throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
+                            "Failed to write data: " + ex);
                 }
-                // nothing can really be done if the file can't be closed
             }
-            if (filename != null) {
-                // if the download wasn't successful, delete the file
-                if (Downloads.Impl.isStatusError(finalStatus)) {
-                    new File(filename).delete();
-                    filename = null;
-                } else if (Downloads.Impl.isStatusSuccess(finalStatus) &&
-                        DrmRawContent.DRM_MIMETYPE_MESSAGE_STRING
-                        .equalsIgnoreCase(mimeType)) {
-                    // transfer the file to the DRM content provider
-                    File file = new File(filename);
-                    Intent item = DrmStore.addDrmFile(mContext.getContentResolver(), file, null);
-                    if (item == null) {
-                        Log.w(Constants.TAG, "unable to add file " + filename + " to DrmProvider");
-                        finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
-                    } else {
-                        filename = item.getDataString();
-                        mimeType = item.getType();
-                    }
+        }
+    }
 
-                    file.delete();
-                } else if (Downloads.Impl.isStatusSuccess(finalStatus)) {
-                    // make sure the file is readable
-                    FileUtils.setPermissions(filename, 0644, -1, -1);
-
-                    // Sync to storage after completion
-                    FileOutputStream downloadedFileStream = null;
-                    try {
-                        downloadedFileStream = new FileOutputStream(filename, true);
-                        downloadedFileStream.getFD().sync();
-                    } catch (FileNotFoundException ex) {
-                        Log.w(Constants.TAG, "file " + filename + " not found: " + ex);
-                    } catch (SyncFailedException ex) {
-                        Log.w(Constants.TAG, "file " + filename + " sync failed: " + ex);
-                    } catch (IOException ex) {
-                        Log.w(Constants.TAG, "IOException trying to sync " + filename + ": " + ex);
-                    } catch (RuntimeException ex) {
-                        Log.w(Constants.TAG, "exception while syncing file: ", ex);
-                    } finally {
-                        if(downloadedFileStream != null) {
-                            try {
-                                downloadedFileStream.close();
-                            } catch (IOException ex) {
-                                Log.w(Constants.TAG, "IOException while closing synced file: ", ex);
-                            } catch (RuntimeException ex) {
-                                Log.w(Constants.TAG, "exception while closing file: ", ex);
-                            }
-                        }
+    /**
+     * Called when we've reached the end of the HTTP response stream, to update the database and
+     * check for consistency.
+     */
+    private void handleEndOfStream(State state) throws StopRequestException {
+        ContentValues values = new ContentValues();
+        values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
+        if (state.mContentLength == -1) {
+            values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, state.mCurrentBytes);
+        }
+        mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
+
+        final boolean lengthMismatched = (state.mContentLength != -1)
+                && (state.mCurrentBytes != state.mContentLength);
+        if (lengthMismatched) {
+            if (cannotResume(state)) {
+                throw new StopRequestException(STATUS_CANNOT_RESUME,
+                        "mismatched content length; unable to resume");
+            } else {
+                throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
+                        "closed socket before end of file");
+            }
+        }
+    }
+
+    private boolean cannotResume(State state) {
+        return (state.mCurrentBytes > 0 && !mInfo.mNoIntegrity && state.mHeaderETag == null)
+                || DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType);
+    }
+
+    /**
+     * Read some data from the HTTP response stream, handling I/O errors.
+     * @param data buffer to use to read data
+     * @param entityStream stream for reading the HTTP response entity
+     * @return the number of bytes actually read or -1 if the end of the stream has been reached
+     */
+    private int readFromResponse(State state, byte[] data, InputStream entityStream)
+            throws StopRequestException {
+        try {
+            return entityStream.read(data);
+        } catch (IOException ex) {
+            // TODO: handle stream errors the same as other retries
+            if ("unexpected end of stream".equals(ex.getMessage())) {
+                return -1;
+            }
+
+            ContentValues values = new ContentValues();
+            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
+            mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
+            if (cannotResume(state)) {
+                throw new StopRequestException(STATUS_CANNOT_RESUME,
+                        "Failed reading response: " + ex + "; unable to resume", ex);
+            } else {
+                throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
+                        "Failed reading response: " + ex, ex);
+            }
+        }
+    }
+
+    /**
+     * Prepare target file based on given network response. Derives filename and
+     * target size as needed.
+     */
+    private void processResponseHeaders(State state, HttpURLConnection conn)
+            throws StopRequestException {
+        // TODO: fallocate the entire file if header gave us specific length
+
+        readResponseHeaders(state, conn);
+
+        state.mFilename = Helpers.generateSaveFile(
+                mContext,
+                mInfo.mUri,
+                mInfo.mHint,
+                state.mContentDisposition,
+                state.mContentLocation,
+                state.mMimeType,
+                mInfo.mDestination,
+                state.mContentLength,
+                mStorageManager);
+
+        updateDatabaseFromHeaders(state);
+        // check connectivity again now that we know the total size
+        checkConnectivity();
+    }
+
+    /**
+     * Update necessary database fields based on values of HTTP response headers that have been
+     * read.
+     */
+    private void updateDatabaseFromHeaders(State state) {
+        ContentValues values = new ContentValues();
+        values.put(Downloads.Impl._DATA, state.mFilename);
+        if (state.mHeaderETag != null) {
+            values.put(Constants.ETAG, state.mHeaderETag);
+        }
+        if (state.mMimeType != null) {
+            values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
+        }
+        values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, mInfo.mTotalBytes);
+        mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
+    }
+
+    /**
+     * Read headers from the HTTP response and store them into local state.
+     */
+    private void readResponseHeaders(State state, HttpURLConnection conn)
+            throws StopRequestException {
+        state.mContentDisposition = conn.getHeaderField("Content-Disposition");
+        state.mContentLocation = conn.getHeaderField("Content-Location");
+
+        if (state.mMimeType == null) {
+            state.mMimeType = Intent.normalizeMimeType(conn.getContentType());
+        }
+
+        state.mHeaderETag = conn.getHeaderField("ETag");
+
+        final String transferEncoding = conn.getHeaderField("Transfer-Encoding");
+        if (transferEncoding == null) {
+            state.mContentLength = getHeaderFieldLong(conn, "Content-Length", -1);
+        } else {
+            Log.i(TAG, "Ignoring Content-Length since Transfer-Encoding is also defined");
+            state.mContentLength = -1;
+        }
+
+        state.mTotalBytes = state.mContentLength;
+        mInfo.mTotalBytes = state.mContentLength;
+
+        final boolean noSizeInfo = state.mContentLength == -1
+                && (transferEncoding == null || !transferEncoding.equalsIgnoreCase("chunked"));
+        if (!mInfo.mNoIntegrity && noSizeInfo) {
+            throw new StopRequestException(STATUS_CANNOT_RESUME,
+                    "can't know size of download, giving up");
+        }
+    }
+
+    private void parseRetryAfterHeaders(State state, HttpURLConnection conn) {
+        state.mRetryAfter = conn.getHeaderFieldInt("Retry-After", -1);
+        if (state.mRetryAfter < 0) {
+            state.mRetryAfter = 0;
+        } else {
+            if (state.mRetryAfter < Constants.MIN_RETRY_AFTER) {
+                state.mRetryAfter = Constants.MIN_RETRY_AFTER;
+            } else if (state.mRetryAfter > Constants.MAX_RETRY_AFTER) {
+                state.mRetryAfter = Constants.MAX_RETRY_AFTER;
+            }
+            state.mRetryAfter += Helpers.sRandom.nextInt(Constants.MIN_RETRY_AFTER + 1);
+            state.mRetryAfter *= 1000;
+        }
+    }
+
+    /**
+     * Prepare the destination file to receive data.  If the file already exists, we'll set up
+     * appropriately for resumption.
+     */
+    private void setupDestinationFile(State state) throws StopRequestException {
+        if (!TextUtils.isEmpty(state.mFilename)) { // only true if we've already run a thread for this download
+            if (Constants.LOGV) {
+                Log.i(Constants.TAG, "have run thread before for id: " + mInfo.mId +
+                        ", and state.mFilename: " + state.mFilename);
+            }
+            if (!Helpers.isFilenameValid(state.mFilename,
+                    mStorageManager.getDownloadDataDirectory())) {
+                // this should never happen
+                throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
+                        "found invalid internal destination filename");
+            }
+            // We're resuming a download that got interrupted
+            File f = new File(state.mFilename);
+            if (f.exists()) {
+                if (Constants.LOGV) {
+                    Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
+                            ", and state.mFilename: " + state.mFilename);
+                }
+                long fileLength = f.length();
+                if (fileLength == 0) {
+                    // The download hadn't actually started, we can restart from scratch
+                    if (Constants.LOGVV) {
+                        Log.d(TAG, "setupDestinationFile() found fileLength=0, deleting "
+                                + state.mFilename);
+                    }
+                    f.delete();
+                    state.mFilename = null;
+                    if (Constants.LOGV) {
+                        Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
+                                ", BUT starting from scratch again: ");
+                    }
+                } else if (mInfo.mETag == null && !mInfo.mNoIntegrity) {
+                    // This should've been caught upon failure
+                    if (Constants.LOGVV) {
+                        Log.d(TAG, "setupDestinationFile() unable to resume download, deleting "
+                                + state.mFilename);
+                    }
+                    f.delete();
+                    throw new StopRequestException(Downloads.Impl.STATUS_CANNOT_RESUME,
+                            "Trying to resume a download that can't be resumed");
+                } else {
+                    // All right, we'll be able to resume this download
+                    if (Constants.LOGV) {
+                        Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
+                                ", and starting with file of length: " + fileLength);
+                    }
+                    state.mCurrentBytes = (int) fileLength;
+                    if (mInfo.mTotalBytes != -1) {
+                        state.mContentLength = mInfo.mTotalBytes;
+                    }
+                    state.mHeaderETag = mInfo.mETag;
+                    state.mContinuingDownload = true;
+                    if (Constants.LOGV) {
+                        Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
+                                ", state.mCurrentBytes: " + state.mCurrentBytes +
+                                ", and setting mContinuingDownload to true: ");
                     }
                 }
             }
-            notifyDownloadCompleted(finalStatus, countRetry, retryAfter, redirectCount,
-                    gotData, filename, newUri, mimeType);
         }
     }
 
     /**
      * Add custom headers for this download to the HTTP request.
      */
-    private void addRequestHeaders(HttpGet request) {
-        for (Map.Entry<String, String> header : mInfo.getHeaders().entrySet()) {
-            request.addHeader(header.getKey(), header.getValue());
+    private void addRequestHeaders(State state, HttpURLConnection conn) {
+        for (Pair<String, String> header : mInfo.getHeaders()) {
+            conn.addRequestProperty(header.first, header.second);
+        }
+
+        // Only splice in user agent when not already defined
+        if (conn.getRequestProperty("User-Agent") == null) {
+            conn.addRequestProperty("User-Agent", userAgent());
+        }
+
+        // Defeat transparent gzip compression, since it doesn't allow us to
+        // easily resume partial downloads.
+        conn.setRequestProperty("Accept-Encoding", "identity");
+
+        if (state.mContinuingDownload) {
+            if (state.mHeaderETag != null) {
+                conn.addRequestProperty("If-Match", state.mHeaderETag);
+            }
+            conn.addRequestProperty("Range", "bytes=" + state.mCurrentBytes + "-");
         }
     }
 
@@ -733,66 +839,76 @@ http_request_loop:
      * Stores information about the completed download, and notifies the initiating application.
      */
     private void notifyDownloadCompleted(
-            int status, boolean countRetry, int retryAfter, int redirectCount, boolean gotData,
-            String filename, String uri, String mimeType) {
-        notifyThroughDatabase(
-                status, countRetry, retryAfter, redirectCount, gotData, filename, uri, mimeType);
-        if (Downloads.Impl.isStatusCompleted(status)) {
-            notifyThroughIntent();
+            State state, int finalStatus, String errorMsg, int numFailed) {
+        notifyThroughDatabase(state, finalStatus, errorMsg, numFailed);
+        if (Downloads.Impl.isStatusCompleted(finalStatus)) {
+            mInfo.sendIntentIfRequested();
         }
     }
 
     private void notifyThroughDatabase(
-            int status, boolean countRetry, int retryAfter, int redirectCount, boolean gotData,
-            String filename, String uri, String mimeType) {
+            State state, int finalStatus, String errorMsg, int numFailed) {
         ContentValues values = new ContentValues();
-        values.put(Downloads.Impl.COLUMN_STATUS, status);
-        values.put(Downloads.Impl._DATA, filename);
-        if (uri != null) {
-            values.put(Downloads.Impl.COLUMN_URI, uri);
-        }
-        values.put(Downloads.Impl.COLUMN_MIME_TYPE, mimeType);
+        values.put(Downloads.Impl.COLUMN_STATUS, finalStatus);
+        values.put(Downloads.Impl._DATA, state.mFilename);
+        values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
         values.put(Downloads.Impl.COLUMN_LAST_MODIFICATION, mSystemFacade.currentTimeMillis());
-        values.put(Constants.RETRY_AFTER_X_REDIRECT_COUNT, retryAfter + (redirectCount << 28));
-        if (!countRetry) {
-            values.put(Constants.FAILED_CONNECTIONS, 0);
-        } else if (gotData) {
-            values.put(Constants.FAILED_CONNECTIONS, 1);
-        } else {
-            values.put(Constants.FAILED_CONNECTIONS, mInfo.mNumFailed + 1);
+        values.put(Downloads.Impl.COLUMN_FAILED_CONNECTIONS, numFailed);
+        values.put(Constants.RETRY_AFTER_X_REDIRECT_COUNT, state.mRetryAfter);
+
+        if (!TextUtils.equals(mInfo.mUri, state.mRequestUri)) {
+            values.put(Downloads.Impl.COLUMN_URI, state.mRequestUri);
         }
 
-        mContext.getContentResolver().update(ContentUris.withAppendedId(
-                Downloads.Impl.CONTENT_URI, mInfo.mId), values, null, null);
+        // save the error message. could be useful to developers.
+        if (!TextUtils.isEmpty(errorMsg)) {
+            values.put(Downloads.Impl.COLUMN_ERROR_MSG, errorMsg);
+        }
+        mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
     }
 
-    /**
-     * Notifies the initiating app if it requested it. That way, it can know that the
-     * download completed even if it's not actively watching the cursor.
-     */
-    private void notifyThroughIntent() {
-        Uri uri = Uri.parse(Downloads.Impl.CONTENT_URI + "/" + mInfo.mId);
-        mInfo.sendIntentIfRequested(uri, mContext);
+    private INetworkPolicyListener mPolicyListener = new INetworkPolicyListener.Stub() {
+        @Override
+        public void onUidRulesChanged(int uid, int uidRules) {
+            // caller is NPMS, since we only register with them
+            if (uid == mInfo.mUid) {
+                mPolicyDirty = true;
+            }
+        }
+
+        @Override
+        public void onMeteredIfacesChanged(String[] meteredIfaces) {
+            // caller is NPMS, since we only register with them
+            mPolicyDirty = true;
+        }
+
+        @Override
+        public void onRestrictBackgroundChanged(boolean restrictBackground) {
+            // caller is NPMS, since we only register with them
+            mPolicyDirty = true;
+        }
+    };
+
+    public static long getHeaderFieldLong(URLConnection conn, String field, long defaultValue) {
+        try {
+            return Long.parseLong(conn.getHeaderField(field));
+        } catch (NumberFormatException e) {
+            return defaultValue;
+        }
     }
 
     /**
-     * Clean up a mimeType string so it can be used to dispatch an intent to
-     * view a downloaded asset.
-     * @param mimeType either null or one or more mime types (semi colon separated).
-     * @return null if mimeType was null. Otherwise a string which represents a
-     * single mimetype in lowercase and with surrounding whitespaces trimmed.
+     * Return if given status is eligible to be treated as
+     * {@link android.provider.Downloads.Impl#STATUS_WAITING_TO_RETRY}.
      */
-    private String sanitizeMimeType(String mimeType) {
-        try {
-            mimeType = mimeType.trim().toLowerCase(Locale.ENGLISH);
-
-            final int semicolonIndex = mimeType.indexOf(';');
-            if (semicolonIndex != -1) {
-                mimeType = mimeType.substring(0, semicolonIndex);
-            }
-            return mimeType;
-        } catch (NullPointerException npe) {
-            return null;
+    public static boolean isStatusRetryable(int status) {
+        switch (status) {
+            case STATUS_HTTP_DATA_ERROR:
+            case HTTP_UNAVAILABLE:
+            case HTTP_INTERNAL_ERROR:
+                return true;
+            default:
+                return false;
         }
     }
 }