Redesign of DownloadManager update loop.
[android/platform/packages/providers/DownloadProvider.git] / src / com / android / providers / downloads / DownloadThread.java
1 /*
2  * Copyright (C) 2008 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 package com.android.providers.downloads;
18
19 import static android.provider.Downloads.Impl.STATUS_BAD_REQUEST;
20 import static android.provider.Downloads.Impl.STATUS_CANNOT_RESUME;
21 import static android.provider.Downloads.Impl.STATUS_FILE_ERROR;
22 import static android.provider.Downloads.Impl.STATUS_HTTP_DATA_ERROR;
23 import static android.provider.Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
24 import static android.provider.Downloads.Impl.STATUS_TOO_MANY_REDIRECTS;
25 import static android.provider.Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
26 import static android.provider.Downloads.Impl.STATUS_WAITING_TO_RETRY;
27 import static android.text.format.DateUtils.MINUTE_IN_MILLIS;
28 import static com.android.providers.downloads.Constants.TAG;
29 import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
30 import static java.net.HttpURLConnection.HTTP_MOVED_PERM;
31 import static java.net.HttpURLConnection.HTTP_MOVED_TEMP;
32 import static java.net.HttpURLConnection.HTTP_OK;
33 import static java.net.HttpURLConnection.HTTP_PARTIAL;
34 import static java.net.HttpURLConnection.HTTP_SEE_OTHER;
35 import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
36
37 import android.content.ContentValues;
38 import android.content.Context;
39 import android.content.Intent;
40 import android.drm.DrmManagerClient;
41 import android.drm.DrmOutputStream;
42 import android.net.INetworkPolicyListener;
43 import android.net.NetworkPolicyManager;
44 import android.net.TrafficStats;
45 import android.os.FileUtils;
46 import android.os.PowerManager;
47 import android.os.Process;
48 import android.os.SystemClock;
49 import android.provider.Downloads;
50 import android.text.TextUtils;
51 import android.util.Log;
52 import android.util.Pair;
53
54 import com.android.providers.downloads.DownloadInfo.NetworkState;
55
56 import java.io.File;
57 import java.io.FileDescriptor;
58 import java.io.FileOutputStream;
59 import java.io.IOException;
60 import java.io.InputStream;
61 import java.io.OutputStream;
62 import java.io.RandomAccessFile;
63 import java.net.HttpURLConnection;
64 import java.net.MalformedURLException;
65 import java.net.URL;
66 import java.net.URLConnection;
67
68 import libcore.io.IoUtils;
69
70 /**
71  * Task which executes a given {@link DownloadInfo}: making network requests,
72  * persisting data to disk, and updating {@link DownloadProvider}.
73  */
74 public class DownloadThread implements Runnable {
75
76     private static final int HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416;
77     private static final int HTTP_TEMP_REDIRECT = 307;
78
79     private static final int DEFAULT_TIMEOUT = (int) MINUTE_IN_MILLIS;
80
81     private final Context mContext;
82     private final DownloadInfo mInfo;
83     private final SystemFacade mSystemFacade;
84     private final StorageManager mStorageManager;
85     private final DownloadNotifier mNotifier;
86
87     private volatile boolean mPolicyDirty;
88
89     public DownloadThread(Context context, SystemFacade systemFacade, DownloadInfo info,
90             StorageManager storageManager, DownloadNotifier notifier) {
91         mContext = context;
92         mSystemFacade = systemFacade;
93         mInfo = info;
94         mStorageManager = storageManager;
95         mNotifier = notifier;
96     }
97
98     /**
99      * Returns the user agent provided by the initiating app, or use the default one
100      */
101     private String userAgent() {
102         String userAgent = mInfo.mUserAgent;
103         if (userAgent == null) {
104             userAgent = Constants.DEFAULT_USER_AGENT;
105         }
106         return userAgent;
107     }
108
109     /**
110      * State for the entire run() method.
111      */
112     static class State {
113         public String mFilename;
114         public String mMimeType;
115         public int mRetryAfter = 0;
116         public boolean mGotData = false;
117         public String mRequestUri;
118         public long mTotalBytes = -1;
119         public long mCurrentBytes = 0;
120         public String mHeaderETag;
121         public boolean mContinuingDownload = false;
122         public long mBytesNotified = 0;
123         public long mTimeLastNotification = 0;
124
125         /** Historical bytes/second speed of this download. */
126         public long mSpeed;
127         /** Time when current sample started. */
128         public long mSpeedSampleStart;
129         /** Bytes transferred since current sample started. */
130         public long mSpeedSampleBytes;
131
132         public long mContentLength = -1;
133         public String mContentDisposition;
134         public String mContentLocation;
135
136         public int mRedirectionCount;
137         public URL mUrl;
138
139         public State(DownloadInfo info) {
140             mMimeType = Intent.normalizeMimeType(info.mMimeType);
141             mRequestUri = info.mUri;
142             mFilename = info.mFileName;
143             mTotalBytes = info.mTotalBytes;
144             mCurrentBytes = info.mCurrentBytes;
145         }
146
147         public void resetBeforeExecute() {
148             // Reset any state from previous execution
149             mContentLength = -1;
150             mContentDisposition = null;
151             mContentLocation = null;
152             mRedirectionCount = 0;
153         }
154     }
155
156     @Override
157     public void run() {
158         Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND);
159         try {
160             runInternal();
161         } finally {
162             mNotifier.notifyDownloadSpeed(mInfo.mId, 0);
163         }
164     }
165
166     private void runInternal() {
167         // Skip when download already marked as finished; this download was
168         // probably started again while racing with UpdateThread.
169         if (DownloadInfo.queryDownloadStatus(mContext.getContentResolver(), mInfo.mId)
170                 == Downloads.Impl.STATUS_SUCCESS) {
171             Log.d(TAG, "Download " + mInfo.mId + " already finished; skipping");
172             return;
173         }
174
175         State state = new State(mInfo);
176         PowerManager.WakeLock wakeLock = null;
177         int finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
178         int numFailed = mInfo.mNumFailed;
179         String errorMsg = null;
180
181         final NetworkPolicyManager netPolicy = NetworkPolicyManager.from(mContext);
182         final PowerManager pm = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE);
183
184         try {
185             wakeLock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, Constants.TAG);
186             wakeLock.acquire();
187
188             // while performing download, register for rules updates
189             netPolicy.registerListener(mPolicyListener);
190
191             Log.i(Constants.TAG, "Initiating download " + mInfo.mId);
192
193             // Network traffic on this thread should be counted against the
194             // requesting UID, and is tagged with well-known value.
195             TrafficStats.setThreadStatsTag(TrafficStats.TAG_SYSTEM_DOWNLOAD);
196             TrafficStats.setThreadStatsUid(mInfo.mUid);
197
198             try {
199                 // TODO: migrate URL sanity checking into client side of API
200                 state.mUrl = new URL(state.mRequestUri);
201             } catch (MalformedURLException e) {
202                 throw new StopRequestException(STATUS_BAD_REQUEST, e);
203             }
204
205             executeDownload(state);
206
207             if (Constants.LOGV) {
208                 Log.v(Constants.TAG, "download completed for " + mInfo.mUri);
209             }
210             finalizeDestinationFile(state);
211             finalStatus = Downloads.Impl.STATUS_SUCCESS;
212         } catch (StopRequestException error) {
213             // remove the cause before printing, in case it contains PII
214             errorMsg = error.getMessage();
215             String msg = "Aborting request for download " + mInfo.mId + ": " + errorMsg;
216             Log.w(Constants.TAG, msg);
217             if (Constants.LOGV) {
218                 Log.w(Constants.TAG, msg, error);
219             }
220             finalStatus = error.getFinalStatus();
221
222             // Nobody below our level should request retries, since we handle
223             // failure counts at this level.
224             if (finalStatus == STATUS_WAITING_TO_RETRY) {
225                 throw new IllegalStateException("Execution should always throw final error codes");
226             }
227
228             // Some errors should be retryable, unless we fail too many times.
229             if (isStatusRetryable(finalStatus)) {
230                 if (state.mGotData) {
231                     numFailed = 1;
232                 } else {
233                     numFailed += 1;
234                 }
235
236                 if (numFailed < Constants.MAX_RETRIES) {
237                     finalStatus = getFinalRetryStatus();
238                 }
239             }
240
241             // fall through to finally block
242         } catch (Throwable ex) {
243             errorMsg = ex.getMessage();
244             String msg = "Exception for id " + mInfo.mId + ": " + errorMsg;
245             Log.w(Constants.TAG, msg, ex);
246             finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
247             // falls through to the code that reports an error
248         } finally {
249             TrafficStats.clearThreadStatsTag();
250             TrafficStats.clearThreadStatsUid();
251
252             cleanupDestination(state, finalStatus);
253             notifyDownloadCompleted(state, finalStatus, errorMsg, numFailed);
254
255             netPolicy.unregisterListener(mPolicyListener);
256
257             if (wakeLock != null) {
258                 wakeLock.release();
259                 wakeLock = null;
260             }
261         }
262         mStorageManager.incrementNumDownloadsSoFar();
263     }
264
265     /**
266      * Fully execute a single download request. Setup and send the request,
267      * handle the response, and transfer the data to the destination file.
268      */
269     private void executeDownload(State state) throws StopRequestException {
270         state.resetBeforeExecute();
271         setupDestinationFile(state);
272
273         // skip when already finished; remove after fixing race in 5217390
274         if (state.mCurrentBytes == state.mTotalBytes) {
275             Log.i(Constants.TAG, "Skipping initiating request for download " +
276                   mInfo.mId + "; already completed");
277             return;
278         }
279
280         while (state.mRedirectionCount++ < Constants.MAX_REDIRECTS) {
281             // Open connection and follow any redirects until we have a useful
282             // response with body.
283             HttpURLConnection conn = null;
284             try {
285                 checkConnectivity();
286                 conn = (HttpURLConnection) state.mUrl.openConnection();
287                 conn.setInstanceFollowRedirects(false);
288                 conn.setConnectTimeout(DEFAULT_TIMEOUT);
289                 conn.setReadTimeout(DEFAULT_TIMEOUT);
290
291                 addRequestHeaders(state, conn);
292
293                 final int responseCode = conn.getResponseCode();
294                 switch (responseCode) {
295                     case HTTP_OK:
296                         if (state.mContinuingDownload) {
297                             throw new StopRequestException(
298                                     STATUS_CANNOT_RESUME, "Expected partial, but received OK");
299                         }
300                         processResponseHeaders(state, conn);
301                         transferData(state, conn);
302                         return;
303
304                     case HTTP_PARTIAL:
305                         if (!state.mContinuingDownload) {
306                             throw new StopRequestException(
307                                     STATUS_CANNOT_RESUME, "Expected OK, but received partial");
308                         }
309                         transferData(state, conn);
310                         return;
311
312                     case HTTP_MOVED_PERM:
313                     case HTTP_MOVED_TEMP:
314                     case HTTP_SEE_OTHER:
315                     case HTTP_TEMP_REDIRECT:
316                         final String location = conn.getHeaderField("Location");
317                         state.mUrl = new URL(state.mUrl, location);
318                         if (responseCode == HTTP_MOVED_PERM) {
319                             // Push updated URL back to database
320                             state.mRequestUri = state.mUrl.toString();
321                         }
322                         continue;
323
324                     case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
325                         throw new StopRequestException(
326                                 STATUS_CANNOT_RESUME, "Requested range not satisfiable");
327
328                     case HTTP_UNAVAILABLE:
329                         parseRetryAfterHeaders(state, conn);
330                         throw new StopRequestException(
331                                 HTTP_UNAVAILABLE, conn.getResponseMessage());
332
333                     case HTTP_INTERNAL_ERROR:
334                         throw new StopRequestException(
335                                 HTTP_INTERNAL_ERROR, conn.getResponseMessage());
336
337                     default:
338                         StopRequestException.throwUnhandledHttpError(
339                                 responseCode, conn.getResponseMessage());
340                 }
341             } catch (IOException e) {
342                 // Trouble with low-level sockets
343                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
344
345             } finally {
346                 if (conn != null) conn.disconnect();
347             }
348         }
349
350         throw new StopRequestException(STATUS_TOO_MANY_REDIRECTS, "Too many redirects");
351     }
352
353     /**
354      * Transfer data from the given connection to the destination file.
355      */
356     private void transferData(State state, HttpURLConnection conn) throws StopRequestException {
357         DrmManagerClient drmClient = null;
358         InputStream in = null;
359         OutputStream out = null;
360         FileDescriptor outFd = null;
361         try {
362             try {
363                 in = conn.getInputStream();
364             } catch (IOException e) {
365                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
366             }
367
368             try {
369                 if (DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType)) {
370                     drmClient = new DrmManagerClient(mContext);
371                     final RandomAccessFile file = new RandomAccessFile(
372                             new File(state.mFilename), "rw");
373                     out = new DrmOutputStream(drmClient, file, state.mMimeType);
374                     outFd = file.getFD();
375                 } else {
376                     out = new FileOutputStream(state.mFilename, true);
377                     outFd = ((FileOutputStream) out).getFD();
378                 }
379             } catch (IOException e) {
380                 throw new StopRequestException(STATUS_FILE_ERROR, e);
381             }
382
383             // Start streaming data, periodically watch for pause/cancel
384             // commands and checking disk space as needed.
385             transferData(state, in, out);
386
387             try {
388                 if (out instanceof DrmOutputStream) {
389                     ((DrmOutputStream) out).finish();
390                 }
391             } catch (IOException e) {
392                 throw new StopRequestException(STATUS_FILE_ERROR, e);
393             }
394
395         } finally {
396             if (drmClient != null) {
397                 drmClient.release();
398             }
399
400             IoUtils.closeQuietly(in);
401
402             try {
403                 if (out != null) out.flush();
404                 if (outFd != null) outFd.sync();
405             } catch (IOException e) {
406             } finally {
407                 IoUtils.closeQuietly(out);
408             }
409         }
410     }
411
412     /**
413      * Check if current connectivity is valid for this request.
414      */
415     private void checkConnectivity() throws StopRequestException {
416         // checking connectivity will apply current policy
417         mPolicyDirty = false;
418
419         final NetworkState networkUsable = mInfo.checkCanUseNetwork();
420         if (networkUsable != NetworkState.OK) {
421             int status = Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
422             if (networkUsable == NetworkState.UNUSABLE_DUE_TO_SIZE) {
423                 status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
424                 mInfo.notifyPauseDueToSize(true);
425             } else if (networkUsable == NetworkState.RECOMMENDED_UNUSABLE_DUE_TO_SIZE) {
426                 status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
427                 mInfo.notifyPauseDueToSize(false);
428             }
429             throw new StopRequestException(status, networkUsable.name());
430         }
431     }
432
433     /**
434      * Return retry status appropriate for current network conditions.
435      */
436     private int getFinalRetryStatus() {
437         switch (mInfo.checkCanUseNetwork()) {
438             case OK:
439                 return STATUS_WAITING_TO_RETRY;
440             case UNUSABLE_DUE_TO_SIZE:
441             case RECOMMENDED_UNUSABLE_DUE_TO_SIZE:
442                 return STATUS_QUEUED_FOR_WIFI;
443             default:
444                 return STATUS_WAITING_FOR_NETWORK;
445         }
446     }
447
448     /**
449      * Transfer as much data as possible from the HTTP response to the
450      * destination file.
451      */
452     private void transferData(State state, InputStream in, OutputStream out)
453             throws StopRequestException {
454         final byte data[] = new byte[Constants.BUFFER_SIZE];
455         for (;;) {
456             int bytesRead = readFromResponse(state, data, in);
457             if (bytesRead == -1) { // success, end of stream already reached
458                 handleEndOfStream(state);
459                 return;
460             }
461
462             state.mGotData = true;
463             writeDataToDestination(state, data, bytesRead, out);
464             state.mCurrentBytes += bytesRead;
465             reportProgress(state);
466
467             if (Constants.LOGVV) {
468                 Log.v(Constants.TAG, "downloaded " + state.mCurrentBytes + " for "
469                       + mInfo.mUri);
470             }
471
472             checkPausedOrCanceled(state);
473         }
474     }
475
476     /**
477      * Called after a successful completion to take any necessary action on the downloaded file.
478      */
479     private void finalizeDestinationFile(State state) {
480         if (state.mFilename != null) {
481             // make sure the file is readable
482             FileUtils.setPermissions(state.mFilename, 0644, -1, -1);
483         }
484     }
485
486     /**
487      * Called just before the thread finishes, regardless of status, to take any necessary action on
488      * the downloaded file.
489      */
490     private void cleanupDestination(State state, int finalStatus) {
491         if (state.mFilename != null && Downloads.Impl.isStatusError(finalStatus)) {
492             if (Constants.LOGVV) {
493                 Log.d(TAG, "cleanupDestination() deleting " + state.mFilename);
494             }
495             new File(state.mFilename).delete();
496             state.mFilename = null;
497         }
498     }
499
500     /**
501      * Check if the download has been paused or canceled, stopping the request appropriately if it
502      * has been.
503      */
504     private void checkPausedOrCanceled(State state) throws StopRequestException {
505         synchronized (mInfo) {
506             if (mInfo.mControl == Downloads.Impl.CONTROL_PAUSED) {
507                 throw new StopRequestException(
508                         Downloads.Impl.STATUS_PAUSED_BY_APP, "download paused by owner");
509             }
510             if (mInfo.mStatus == Downloads.Impl.STATUS_CANCELED) {
511                 throw new StopRequestException(Downloads.Impl.STATUS_CANCELED, "download canceled");
512             }
513         }
514
515         // if policy has been changed, trigger connectivity check
516         if (mPolicyDirty) {
517             checkConnectivity();
518         }
519     }
520
521     /**
522      * Report download progress through the database if necessary.
523      */
524     private void reportProgress(State state) {
525         final long now = SystemClock.elapsedRealtime();
526
527         final long sampleDelta = now - state.mSpeedSampleStart;
528         if (sampleDelta > 500) {
529             final long sampleSpeed = ((state.mCurrentBytes - state.mSpeedSampleBytes) * 1000)
530                     / sampleDelta;
531
532             if (state.mSpeed == 0) {
533                 state.mSpeed = sampleSpeed;
534             } else {
535                 state.mSpeed = ((state.mSpeed * 3) + sampleSpeed) / 4;
536             }
537
538             state.mSpeedSampleStart = now;
539             state.mSpeedSampleBytes = state.mCurrentBytes;
540
541             mNotifier.notifyDownloadSpeed(mInfo.mId, state.mSpeed);
542         }
543
544         if (state.mCurrentBytes - state.mBytesNotified > Constants.MIN_PROGRESS_STEP &&
545             now - state.mTimeLastNotification > Constants.MIN_PROGRESS_TIME) {
546             ContentValues values = new ContentValues();
547             values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
548             mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
549             state.mBytesNotified = state.mCurrentBytes;
550             state.mTimeLastNotification = now;
551         }
552     }
553
554     /**
555      * Write a data buffer to the destination file.
556      * @param data buffer containing the data to write
557      * @param bytesRead how many bytes to write from the buffer
558      */
559     private void writeDataToDestination(State state, byte[] data, int bytesRead, OutputStream out)
560             throws StopRequestException {
561         mStorageManager.verifySpaceBeforeWritingToFile(
562                 mInfo.mDestination, state.mFilename, bytesRead);
563
564         boolean forceVerified = false;
565         while (true) {
566             try {
567                 out.write(data, 0, bytesRead);
568                 return;
569             } catch (IOException ex) {
570                 // TODO: better differentiate between DRM and disk failures
571                 if (!forceVerified) {
572                     // couldn't write to file. are we out of space? check.
573                     mStorageManager.verifySpace(mInfo.mDestination, state.mFilename, bytesRead);
574                     forceVerified = true;
575                 } else {
576                     throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
577                             "Failed to write data: " + ex);
578                 }
579             }
580         }
581     }
582
583     /**
584      * Called when we've reached the end of the HTTP response stream, to update the database and
585      * check for consistency.
586      */
587     private void handleEndOfStream(State state) throws StopRequestException {
588         ContentValues values = new ContentValues();
589         values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
590         if (state.mContentLength == -1) {
591             values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, state.mCurrentBytes);
592         }
593         mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
594
595         final boolean lengthMismatched = (state.mContentLength != -1)
596                 && (state.mCurrentBytes != state.mContentLength);
597         if (lengthMismatched) {
598             if (cannotResume(state)) {
599                 throw new StopRequestException(STATUS_CANNOT_RESUME,
600                         "mismatched content length; unable to resume");
601             } else {
602                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
603                         "closed socket before end of file");
604             }
605         }
606     }
607
608     private boolean cannotResume(State state) {
609         return (state.mCurrentBytes > 0 && !mInfo.mNoIntegrity && state.mHeaderETag == null)
610                 || DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType);
611     }
612
613     /**
614      * Read some data from the HTTP response stream, handling I/O errors.
615      * @param data buffer to use to read data
616      * @param entityStream stream for reading the HTTP response entity
617      * @return the number of bytes actually read or -1 if the end of the stream has been reached
618      */
619     private int readFromResponse(State state, byte[] data, InputStream entityStream)
620             throws StopRequestException {
621         try {
622             return entityStream.read(data);
623         } catch (IOException ex) {
624             // TODO: handle stream errors the same as other retries
625             if ("unexpected end of stream".equals(ex.getMessage())) {
626                 return -1;
627             }
628
629             ContentValues values = new ContentValues();
630             values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
631             mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
632             if (cannotResume(state)) {
633                 throw new StopRequestException(STATUS_CANNOT_RESUME,
634                         "Failed reading response: " + ex + "; unable to resume", ex);
635             } else {
636                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
637                         "Failed reading response: " + ex, ex);
638             }
639         }
640     }
641
642     /**
643      * Prepare target file based on given network response. Derives filename and
644      * target size as needed.
645      */
646     private void processResponseHeaders(State state, HttpURLConnection conn)
647             throws StopRequestException {
648         // TODO: fallocate the entire file if header gave us specific length
649
650         readResponseHeaders(state, conn);
651
652         state.mFilename = Helpers.generateSaveFile(
653                 mContext,
654                 mInfo.mUri,
655                 mInfo.mHint,
656                 state.mContentDisposition,
657                 state.mContentLocation,
658                 state.mMimeType,
659                 mInfo.mDestination,
660                 state.mContentLength,
661                 mStorageManager);
662
663         updateDatabaseFromHeaders(state);
664         // check connectivity again now that we know the total size
665         checkConnectivity();
666     }
667
668     /**
669      * Update necessary database fields based on values of HTTP response headers that have been
670      * read.
671      */
672     private void updateDatabaseFromHeaders(State state) {
673         ContentValues values = new ContentValues();
674         values.put(Downloads.Impl._DATA, state.mFilename);
675         if (state.mHeaderETag != null) {
676             values.put(Constants.ETAG, state.mHeaderETag);
677         }
678         if (state.mMimeType != null) {
679             values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
680         }
681         values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, mInfo.mTotalBytes);
682         mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
683     }
684
685     /**
686      * Read headers from the HTTP response and store them into local state.
687      */
688     private void readResponseHeaders(State state, HttpURLConnection conn)
689             throws StopRequestException {
690         state.mContentDisposition = conn.getHeaderField("Content-Disposition");
691         state.mContentLocation = conn.getHeaderField("Content-Location");
692
693         if (state.mMimeType == null) {
694             state.mMimeType = Intent.normalizeMimeType(conn.getContentType());
695         }
696
697         state.mHeaderETag = conn.getHeaderField("ETag");
698
699         final String transferEncoding = conn.getHeaderField("Transfer-Encoding");
700         if (transferEncoding == null) {
701             state.mContentLength = getHeaderFieldLong(conn, "Content-Length", -1);
702         } else {
703             Log.i(TAG, "Ignoring Content-Length since Transfer-Encoding is also defined");
704             state.mContentLength = -1;
705         }
706
707         state.mTotalBytes = state.mContentLength;
708         mInfo.mTotalBytes = state.mContentLength;
709
710         final boolean noSizeInfo = state.mContentLength == -1
711                 && (transferEncoding == null || !transferEncoding.equalsIgnoreCase("chunked"));
712         if (!mInfo.mNoIntegrity && noSizeInfo) {
713             throw new StopRequestException(STATUS_CANNOT_RESUME,
714                     "can't know size of download, giving up");
715         }
716     }
717
718     private void parseRetryAfterHeaders(State state, HttpURLConnection conn) {
719         state.mRetryAfter = conn.getHeaderFieldInt("Retry-After", -1);
720         if (state.mRetryAfter < 0) {
721             state.mRetryAfter = 0;
722         } else {
723             if (state.mRetryAfter < Constants.MIN_RETRY_AFTER) {
724                 state.mRetryAfter = Constants.MIN_RETRY_AFTER;
725             } else if (state.mRetryAfter > Constants.MAX_RETRY_AFTER) {
726                 state.mRetryAfter = Constants.MAX_RETRY_AFTER;
727             }
728             state.mRetryAfter += Helpers.sRandom.nextInt(Constants.MIN_RETRY_AFTER + 1);
729             state.mRetryAfter *= 1000;
730         }
731     }
732
733     /**
734      * Prepare the destination file to receive data.  If the file already exists, we'll set up
735      * appropriately for resumption.
736      */
737     private void setupDestinationFile(State state) throws StopRequestException {
738         if (!TextUtils.isEmpty(state.mFilename)) { // only true if we've already run a thread for this download
739             if (Constants.LOGV) {
740                 Log.i(Constants.TAG, "have run thread before for id: " + mInfo.mId +
741                         ", and state.mFilename: " + state.mFilename);
742             }
743             if (!Helpers.isFilenameValid(state.mFilename,
744                     mStorageManager.getDownloadDataDirectory())) {
745                 // this should never happen
746                 throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
747                         "found invalid internal destination filename");
748             }
749             // We're resuming a download that got interrupted
750             File f = new File(state.mFilename);
751             if (f.exists()) {
752                 if (Constants.LOGV) {
753                     Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
754                             ", and state.mFilename: " + state.mFilename);
755                 }
756                 long fileLength = f.length();
757                 if (fileLength == 0) {
758                     // The download hadn't actually started, we can restart from scratch
759                     if (Constants.LOGVV) {
760                         Log.d(TAG, "setupDestinationFile() found fileLength=0, deleting "
761                                 + state.mFilename);
762                     }
763                     f.delete();
764                     state.mFilename = null;
765                     if (Constants.LOGV) {
766                         Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
767                                 ", BUT starting from scratch again: ");
768                     }
769                 } else if (mInfo.mETag == null && !mInfo.mNoIntegrity) {
770                     // This should've been caught upon failure
771                     if (Constants.LOGVV) {
772                         Log.d(TAG, "setupDestinationFile() unable to resume download, deleting "
773                                 + state.mFilename);
774                     }
775                     f.delete();
776                     throw new StopRequestException(Downloads.Impl.STATUS_CANNOT_RESUME,
777                             "Trying to resume a download that can't be resumed");
778                 } else {
779                     // All right, we'll be able to resume this download
780                     if (Constants.LOGV) {
781                         Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
782                                 ", and starting with file of length: " + fileLength);
783                     }
784                     state.mCurrentBytes = (int) fileLength;
785                     if (mInfo.mTotalBytes != -1) {
786                         state.mContentLength = mInfo.mTotalBytes;
787                     }
788                     state.mHeaderETag = mInfo.mETag;
789                     state.mContinuingDownload = true;
790                     if (Constants.LOGV) {
791                         Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
792                                 ", state.mCurrentBytes: " + state.mCurrentBytes +
793                                 ", and setting mContinuingDownload to true: ");
794                     }
795                 }
796             }
797         }
798     }
799
800     /**
801      * Add custom headers for this download to the HTTP request.
802      */
803     private void addRequestHeaders(State state, HttpURLConnection conn) {
804         for (Pair<String, String> header : mInfo.getHeaders()) {
805             conn.addRequestProperty(header.first, header.second);
806         }
807
808         // Only splice in user agent when not already defined
809         if (conn.getRequestProperty("User-Agent") == null) {
810             conn.addRequestProperty("User-Agent", userAgent());
811         }
812
813         if (state.mContinuingDownload) {
814             if (state.mHeaderETag != null) {
815                 conn.addRequestProperty("If-Match", state.mHeaderETag);
816             }
817             conn.addRequestProperty("Range", "bytes=" + state.mCurrentBytes + "-");
818         }
819     }
820
821     /**
822      * Stores information about the completed download, and notifies the initiating application.
823      */
824     private void notifyDownloadCompleted(
825             State state, int finalStatus, String errorMsg, int numFailed) {
826         if (Downloads.Impl.isStatusCompleted(finalStatus)) {
827             mInfo.sendIntentIfRequested();
828         }
829         notifyThroughDatabase(state, finalStatus, errorMsg, numFailed);
830     }
831
832     private void notifyThroughDatabase(
833             State state, int finalStatus, String errorMsg, int numFailed) {
834         ContentValues values = new ContentValues();
835         values.put(Downloads.Impl.COLUMN_STATUS, finalStatus);
836         values.put(Downloads.Impl._DATA, state.mFilename);
837         values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
838         values.put(Downloads.Impl.COLUMN_LAST_MODIFICATION, mSystemFacade.currentTimeMillis());
839         values.put(Downloads.Impl.COLUMN_FAILED_CONNECTIONS, numFailed);
840         values.put(Constants.RETRY_AFTER_X_REDIRECT_COUNT, state.mRetryAfter);
841
842         if (!TextUtils.equals(mInfo.mUri, state.mRequestUri)) {
843             values.put(Downloads.Impl.COLUMN_URI, state.mRequestUri);
844         }
845
846         // save the error message. could be useful to developers.
847         if (!TextUtils.isEmpty(errorMsg)) {
848             values.put(Downloads.Impl.COLUMN_ERROR_MSG, errorMsg);
849         }
850         mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
851     }
852
853     private INetworkPolicyListener mPolicyListener = new INetworkPolicyListener.Stub() {
854         @Override
855         public void onUidRulesChanged(int uid, int uidRules) {
856             // caller is NPMS, since we only register with them
857             if (uid == mInfo.mUid) {
858                 mPolicyDirty = true;
859             }
860         }
861
862         @Override
863         public void onMeteredIfacesChanged(String[] meteredIfaces) {
864             // caller is NPMS, since we only register with them
865             mPolicyDirty = true;
866         }
867
868         @Override
869         public void onRestrictBackgroundChanged(boolean restrictBackground) {
870             // caller is NPMS, since we only register with them
871             mPolicyDirty = true;
872         }
873     };
874
875     public static long getHeaderFieldLong(URLConnection conn, String field, long defaultValue) {
876         try {
877             return Long.parseLong(conn.getHeaderField(field));
878         } catch (NumberFormatException e) {
879             return defaultValue;
880         }
881     }
882
883     /**
884      * Return if given status is eligible to be treated as
885      * {@link android.provider.Downloads.Impl#STATUS_WAITING_TO_RETRY}.
886      */
887     public static boolean isStatusRetryable(int status) {
888         switch (status) {
889             case STATUS_HTTP_DATA_ERROR:
890             case HTTP_UNAVAILABLE:
891             case HTTP_INTERNAL_ERROR:
892                 return true;
893             default:
894                 return false;
895         }
896     }
897 }