Increment operation counts to track downloads.
[android/platform/packages/providers/DownloadProvider.git] / src / com / android / providers / downloads / DownloadThread.java
1 /*
2  * Copyright (C) 2008 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16
17 package com.android.providers.downloads;
18
19 import static android.provider.Downloads.Impl.STATUS_BAD_REQUEST;
20 import static android.provider.Downloads.Impl.STATUS_CANNOT_RESUME;
21 import static android.provider.Downloads.Impl.STATUS_FILE_ERROR;
22 import static android.provider.Downloads.Impl.STATUS_HTTP_DATA_ERROR;
23 import static android.provider.Downloads.Impl.STATUS_SUCCESS;
24 import static android.provider.Downloads.Impl.STATUS_TOO_MANY_REDIRECTS;
25 import static android.provider.Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
26 import static android.provider.Downloads.Impl.STATUS_WAITING_TO_RETRY;
27 import static android.text.format.DateUtils.SECOND_IN_MILLIS;
28 import static com.android.providers.downloads.Constants.TAG;
29 import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
30 import static java.net.HttpURLConnection.HTTP_MOVED_PERM;
31 import static java.net.HttpURLConnection.HTTP_MOVED_TEMP;
32 import static java.net.HttpURLConnection.HTTP_OK;
33 import static java.net.HttpURLConnection.HTTP_PARTIAL;
34 import static java.net.HttpURLConnection.HTTP_SEE_OTHER;
35 import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
36
37 import android.content.ContentValues;
38 import android.content.Context;
39 import android.content.Intent;
40 import android.drm.DrmManagerClient;
41 import android.drm.DrmOutputStream;
42 import android.net.ConnectivityManager;
43 import android.net.INetworkPolicyListener;
44 import android.net.NetworkInfo;
45 import android.net.NetworkPolicyManager;
46 import android.net.TrafficStats;
47 import android.os.FileUtils;
48 import android.os.PowerManager;
49 import android.os.Process;
50 import android.os.SystemClock;
51 import android.provider.Downloads;
52 import android.text.TextUtils;
53 import android.util.Log;
54 import android.util.Pair;
55
56 import com.android.providers.downloads.DownloadInfo.NetworkState;
57
58 import libcore.io.IoUtils;
59
60 import java.io.File;
61 import java.io.FileDescriptor;
62 import java.io.FileOutputStream;
63 import java.io.IOException;
64 import java.io.InputStream;
65 import java.io.OutputStream;
66 import java.io.RandomAccessFile;
67 import java.net.HttpURLConnection;
68 import java.net.MalformedURLException;
69 import java.net.URL;
70 import java.net.URLConnection;
71
72 /**
73  * Task which executes a given {@link DownloadInfo}: making network requests,
74  * persisting data to disk, and updating {@link DownloadProvider}.
75  */
76 public class DownloadThread implements Runnable {
77
78     // TODO: bind each download to a specific network interface to avoid state
79     // checking races once we have ConnectivityManager API
80
81     private static final int HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416;
82     private static final int HTTP_TEMP_REDIRECT = 307;
83
84     private static final int DEFAULT_TIMEOUT = (int) (20 * SECOND_IN_MILLIS);
85
86     private final Context mContext;
87     private final DownloadInfo mInfo;
88     private final SystemFacade mSystemFacade;
89     private final StorageManager mStorageManager;
90     private final DownloadNotifier mNotifier;
91
92     private volatile boolean mPolicyDirty;
93
94     public DownloadThread(Context context, SystemFacade systemFacade, DownloadInfo info,
95             StorageManager storageManager, DownloadNotifier notifier) {
96         mContext = context;
97         mSystemFacade = systemFacade;
98         mInfo = info;
99         mStorageManager = storageManager;
100         mNotifier = notifier;
101     }
102
103     /**
104      * Returns the user agent provided by the initiating app, or use the default one
105      */
106     private String userAgent() {
107         String userAgent = mInfo.mUserAgent;
108         if (userAgent == null) {
109             userAgent = Constants.DEFAULT_USER_AGENT;
110         }
111         return userAgent;
112     }
113
114     /**
115      * State for the entire run() method.
116      */
117     static class State {
118         public String mFilename;
119         public String mMimeType;
120         public int mRetryAfter = 0;
121         public boolean mGotData = false;
122         public String mRequestUri;
123         public long mTotalBytes = -1;
124         public long mCurrentBytes = 0;
125         public String mHeaderETag;
126         public boolean mContinuingDownload = false;
127         public long mBytesNotified = 0;
128         public long mTimeLastNotification = 0;
129         public int mNetworkType = ConnectivityManager.TYPE_NONE;
130
131         /** Historical bytes/second speed of this download. */
132         public long mSpeed;
133         /** Time when current sample started. */
134         public long mSpeedSampleStart;
135         /** Bytes transferred since current sample started. */
136         public long mSpeedSampleBytes;
137
138         public long mContentLength = -1;
139         public String mContentDisposition;
140         public String mContentLocation;
141
142         public int mRedirectionCount;
143         public URL mUrl;
144
145         public State(DownloadInfo info) {
146             mMimeType = Intent.normalizeMimeType(info.mMimeType);
147             mRequestUri = info.mUri;
148             mFilename = info.mFileName;
149             mTotalBytes = info.mTotalBytes;
150             mCurrentBytes = info.mCurrentBytes;
151         }
152
153         public void resetBeforeExecute() {
154             // Reset any state from previous execution
155             mContentLength = -1;
156             mContentDisposition = null;
157             mContentLocation = null;
158             mRedirectionCount = 0;
159         }
160     }
161
162     @Override
163     public void run() {
164         Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND);
165         try {
166             runInternal();
167         } finally {
168             mNotifier.notifyDownloadSpeed(mInfo.mId, 0);
169         }
170     }
171
172     private void runInternal() {
173         // Skip when download already marked as finished; this download was
174         // probably started again while racing with UpdateThread.
175         if (DownloadInfo.queryDownloadStatus(mContext.getContentResolver(), mInfo.mId)
176                 == Downloads.Impl.STATUS_SUCCESS) {
177             Log.d(TAG, "Download " + mInfo.mId + " already finished; skipping");
178             return;
179         }
180
181         State state = new State(mInfo);
182         PowerManager.WakeLock wakeLock = null;
183         int finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
184         int numFailed = mInfo.mNumFailed;
185         String errorMsg = null;
186
187         final NetworkPolicyManager netPolicy = NetworkPolicyManager.from(mContext);
188         final PowerManager pm = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE);
189
190         try {
191             wakeLock = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, Constants.TAG);
192             wakeLock.acquire();
193
194             // while performing download, register for rules updates
195             netPolicy.registerListener(mPolicyListener);
196
197             Log.i(Constants.TAG, "Download " + mInfo.mId + " starting");
198
199             // Remember which network this download started on; used to
200             // determine if errors were due to network changes.
201             final NetworkInfo info = mSystemFacade.getActiveNetworkInfo(mInfo.mUid);
202             if (info != null) {
203                 state.mNetworkType = info.getType();
204             }
205
206             // Network traffic on this thread should be counted against the
207             // requesting UID, and is tagged with well-known value.
208             TrafficStats.setThreadStatsTag(TrafficStats.TAG_SYSTEM_DOWNLOAD);
209             TrafficStats.setThreadStatsUid(mInfo.mUid);
210
211             try {
212                 // TODO: migrate URL sanity checking into client side of API
213                 state.mUrl = new URL(state.mRequestUri);
214             } catch (MalformedURLException e) {
215                 throw new StopRequestException(STATUS_BAD_REQUEST, e);
216             }
217
218             executeDownload(state);
219
220             finalizeDestinationFile(state);
221             finalStatus = Downloads.Impl.STATUS_SUCCESS;
222         } catch (StopRequestException error) {
223             // remove the cause before printing, in case it contains PII
224             errorMsg = error.getMessage();
225             String msg = "Aborting request for download " + mInfo.mId + ": " + errorMsg;
226             Log.w(Constants.TAG, msg);
227             if (Constants.LOGV) {
228                 Log.w(Constants.TAG, msg, error);
229             }
230             finalStatus = error.getFinalStatus();
231
232             // Nobody below our level should request retries, since we handle
233             // failure counts at this level.
234             if (finalStatus == STATUS_WAITING_TO_RETRY) {
235                 throw new IllegalStateException("Execution should always throw final error codes");
236             }
237
238             // Some errors should be retryable, unless we fail too many times.
239             if (isStatusRetryable(finalStatus)) {
240                 if (state.mGotData) {
241                     numFailed = 1;
242                 } else {
243                     numFailed += 1;
244                 }
245
246                 if (numFailed < Constants.MAX_RETRIES) {
247                     final NetworkInfo info = mSystemFacade.getActiveNetworkInfo(mInfo.mUid);
248                     if (info != null && info.getType() == state.mNetworkType
249                             && info.isConnected()) {
250                         // Underlying network is still intact, use normal backoff
251                         finalStatus = STATUS_WAITING_TO_RETRY;
252                     } else {
253                         // Network changed, retry on any next available
254                         finalStatus = STATUS_WAITING_FOR_NETWORK;
255                     }
256                 }
257             }
258
259             // fall through to finally block
260         } catch (Throwable ex) {
261             errorMsg = ex.getMessage();
262             String msg = "Exception for id " + mInfo.mId + ": " + errorMsg;
263             Log.w(Constants.TAG, msg, ex);
264             finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
265             // falls through to the code that reports an error
266         } finally {
267             if (finalStatus == STATUS_SUCCESS) {
268                 TrafficStats.incrementOperationCount(1);
269             }
270
271             TrafficStats.clearThreadStatsTag();
272             TrafficStats.clearThreadStatsUid();
273
274             cleanupDestination(state, finalStatus);
275             notifyDownloadCompleted(state, finalStatus, errorMsg, numFailed);
276
277             Log.i(Constants.TAG, "Download " + mInfo.mId + " finished with status "
278                     + Downloads.Impl.statusToString(finalStatus));
279
280             netPolicy.unregisterListener(mPolicyListener);
281
282             if (wakeLock != null) {
283                 wakeLock.release();
284                 wakeLock = null;
285             }
286         }
287         mStorageManager.incrementNumDownloadsSoFar();
288     }
289
290     /**
291      * Fully execute a single download request. Setup and send the request,
292      * handle the response, and transfer the data to the destination file.
293      */
294     private void executeDownload(State state) throws StopRequestException {
295         state.resetBeforeExecute();
296         setupDestinationFile(state);
297
298         // skip when already finished; remove after fixing race in 5217390
299         if (state.mCurrentBytes == state.mTotalBytes) {
300             Log.i(Constants.TAG, "Skipping initiating request for download " +
301                   mInfo.mId + "; already completed");
302             return;
303         }
304
305         while (state.mRedirectionCount++ < Constants.MAX_REDIRECTS) {
306             // Open connection and follow any redirects until we have a useful
307             // response with body.
308             HttpURLConnection conn = null;
309             try {
310                 checkConnectivity();
311                 conn = (HttpURLConnection) state.mUrl.openConnection();
312                 conn.setInstanceFollowRedirects(false);
313                 conn.setConnectTimeout(DEFAULT_TIMEOUT);
314                 conn.setReadTimeout(DEFAULT_TIMEOUT);
315
316                 addRequestHeaders(state, conn);
317
318                 final int responseCode = conn.getResponseCode();
319                 switch (responseCode) {
320                     case HTTP_OK:
321                         if (state.mContinuingDownload) {
322                             throw new StopRequestException(
323                                     STATUS_CANNOT_RESUME, "Expected partial, but received OK");
324                         }
325                         processResponseHeaders(state, conn);
326                         transferData(state, conn);
327                         return;
328
329                     case HTTP_PARTIAL:
330                         if (!state.mContinuingDownload) {
331                             throw new StopRequestException(
332                                     STATUS_CANNOT_RESUME, "Expected OK, but received partial");
333                         }
334                         transferData(state, conn);
335                         return;
336
337                     case HTTP_MOVED_PERM:
338                     case HTTP_MOVED_TEMP:
339                     case HTTP_SEE_OTHER:
340                     case HTTP_TEMP_REDIRECT:
341                         final String location = conn.getHeaderField("Location");
342                         state.mUrl = new URL(state.mUrl, location);
343                         if (responseCode == HTTP_MOVED_PERM) {
344                             // Push updated URL back to database
345                             state.mRequestUri = state.mUrl.toString();
346                         }
347                         continue;
348
349                     case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
350                         throw new StopRequestException(
351                                 STATUS_CANNOT_RESUME, "Requested range not satisfiable");
352
353                     case HTTP_UNAVAILABLE:
354                         parseRetryAfterHeaders(state, conn);
355                         throw new StopRequestException(
356                                 HTTP_UNAVAILABLE, conn.getResponseMessage());
357
358                     case HTTP_INTERNAL_ERROR:
359                         throw new StopRequestException(
360                                 HTTP_INTERNAL_ERROR, conn.getResponseMessage());
361
362                     default:
363                         StopRequestException.throwUnhandledHttpError(
364                                 responseCode, conn.getResponseMessage());
365                 }
366             } catch (IOException e) {
367                 // Trouble with low-level sockets
368                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
369
370             } finally {
371                 if (conn != null) conn.disconnect();
372             }
373         }
374
375         throw new StopRequestException(STATUS_TOO_MANY_REDIRECTS, "Too many redirects");
376     }
377
378     /**
379      * Transfer data from the given connection to the destination file.
380      */
381     private void transferData(State state, HttpURLConnection conn) throws StopRequestException {
382         DrmManagerClient drmClient = null;
383         InputStream in = null;
384         OutputStream out = null;
385         FileDescriptor outFd = null;
386         try {
387             try {
388                 in = conn.getInputStream();
389             } catch (IOException e) {
390                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
391             }
392
393             try {
394                 if (DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType)) {
395                     drmClient = new DrmManagerClient(mContext);
396                     final RandomAccessFile file = new RandomAccessFile(
397                             new File(state.mFilename), "rw");
398                     out = new DrmOutputStream(drmClient, file, state.mMimeType);
399                     outFd = file.getFD();
400                 } else {
401                     out = new FileOutputStream(state.mFilename, true);
402                     outFd = ((FileOutputStream) out).getFD();
403                 }
404             } catch (IOException e) {
405                 throw new StopRequestException(STATUS_FILE_ERROR, e);
406             }
407
408             // Start streaming data, periodically watch for pause/cancel
409             // commands and checking disk space as needed.
410             transferData(state, in, out);
411
412             try {
413                 if (out instanceof DrmOutputStream) {
414                     ((DrmOutputStream) out).finish();
415                 }
416             } catch (IOException e) {
417                 throw new StopRequestException(STATUS_FILE_ERROR, e);
418             }
419
420         } finally {
421             if (drmClient != null) {
422                 drmClient.release();
423             }
424
425             IoUtils.closeQuietly(in);
426
427             try {
428                 if (out != null) out.flush();
429                 if (outFd != null) outFd.sync();
430             } catch (IOException e) {
431             } finally {
432                 IoUtils.closeQuietly(out);
433             }
434         }
435     }
436
437     /**
438      * Check if current connectivity is valid for this request.
439      */
440     private void checkConnectivity() throws StopRequestException {
441         // checking connectivity will apply current policy
442         mPolicyDirty = false;
443
444         final NetworkState networkUsable = mInfo.checkCanUseNetwork();
445         if (networkUsable != NetworkState.OK) {
446             int status = Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
447             if (networkUsable == NetworkState.UNUSABLE_DUE_TO_SIZE) {
448                 status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
449                 mInfo.notifyPauseDueToSize(true);
450             } else if (networkUsable == NetworkState.RECOMMENDED_UNUSABLE_DUE_TO_SIZE) {
451                 status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
452                 mInfo.notifyPauseDueToSize(false);
453             }
454             throw new StopRequestException(status, networkUsable.name());
455         }
456     }
457
458     /**
459      * Transfer as much data as possible from the HTTP response to the
460      * destination file.
461      */
462     private void transferData(State state, InputStream in, OutputStream out)
463             throws StopRequestException {
464         final byte data[] = new byte[Constants.BUFFER_SIZE];
465         for (;;) {
466             int bytesRead = readFromResponse(state, data, in);
467             if (bytesRead == -1) { // success, end of stream already reached
468                 handleEndOfStream(state);
469                 return;
470             }
471
472             state.mGotData = true;
473             writeDataToDestination(state, data, bytesRead, out);
474             state.mCurrentBytes += bytesRead;
475             reportProgress(state);
476
477             if (Constants.LOGVV) {
478                 Log.v(Constants.TAG, "downloaded " + state.mCurrentBytes + " for "
479                       + mInfo.mUri);
480             }
481
482             checkPausedOrCanceled(state);
483         }
484     }
485
486     /**
487      * Called after a successful completion to take any necessary action on the downloaded file.
488      */
489     private void finalizeDestinationFile(State state) {
490         if (state.mFilename != null) {
491             // make sure the file is readable
492             FileUtils.setPermissions(state.mFilename, 0644, -1, -1);
493         }
494     }
495
496     /**
497      * Called just before the thread finishes, regardless of status, to take any necessary action on
498      * the downloaded file.
499      */
500     private void cleanupDestination(State state, int finalStatus) {
501         if (state.mFilename != null && Downloads.Impl.isStatusError(finalStatus)) {
502             if (Constants.LOGVV) {
503                 Log.d(TAG, "cleanupDestination() deleting " + state.mFilename);
504             }
505             new File(state.mFilename).delete();
506             state.mFilename = null;
507         }
508     }
509
510     /**
511      * Check if the download has been paused or canceled, stopping the request appropriately if it
512      * has been.
513      */
514     private void checkPausedOrCanceled(State state) throws StopRequestException {
515         synchronized (mInfo) {
516             if (mInfo.mControl == Downloads.Impl.CONTROL_PAUSED) {
517                 throw new StopRequestException(
518                         Downloads.Impl.STATUS_PAUSED_BY_APP, "download paused by owner");
519             }
520             if (mInfo.mStatus == Downloads.Impl.STATUS_CANCELED) {
521                 throw new StopRequestException(Downloads.Impl.STATUS_CANCELED, "download canceled");
522             }
523         }
524
525         // if policy has been changed, trigger connectivity check
526         if (mPolicyDirty) {
527             checkConnectivity();
528         }
529     }
530
531     /**
532      * Report download progress through the database if necessary.
533      */
534     private void reportProgress(State state) {
535         final long now = SystemClock.elapsedRealtime();
536
537         final long sampleDelta = now - state.mSpeedSampleStart;
538         if (sampleDelta > 500) {
539             final long sampleSpeed = ((state.mCurrentBytes - state.mSpeedSampleBytes) * 1000)
540                     / sampleDelta;
541
542             if (state.mSpeed == 0) {
543                 state.mSpeed = sampleSpeed;
544             } else {
545                 state.mSpeed = ((state.mSpeed * 3) + sampleSpeed) / 4;
546             }
547
548             // Only notify once we have a full sample window
549             if (state.mSpeedSampleStart != 0) {
550                 mNotifier.notifyDownloadSpeed(mInfo.mId, state.mSpeed);
551             }
552
553             state.mSpeedSampleStart = now;
554             state.mSpeedSampleBytes = state.mCurrentBytes;
555         }
556
557         if (state.mCurrentBytes - state.mBytesNotified > Constants.MIN_PROGRESS_STEP &&
558             now - state.mTimeLastNotification > Constants.MIN_PROGRESS_TIME) {
559             ContentValues values = new ContentValues();
560             values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
561             mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
562             state.mBytesNotified = state.mCurrentBytes;
563             state.mTimeLastNotification = now;
564         }
565     }
566
567     /**
568      * Write a data buffer to the destination file.
569      * @param data buffer containing the data to write
570      * @param bytesRead how many bytes to write from the buffer
571      */
572     private void writeDataToDestination(State state, byte[] data, int bytesRead, OutputStream out)
573             throws StopRequestException {
574         mStorageManager.verifySpaceBeforeWritingToFile(
575                 mInfo.mDestination, state.mFilename, bytesRead);
576
577         boolean forceVerified = false;
578         while (true) {
579             try {
580                 out.write(data, 0, bytesRead);
581                 return;
582             } catch (IOException ex) {
583                 // TODO: better differentiate between DRM and disk failures
584                 if (!forceVerified) {
585                     // couldn't write to file. are we out of space? check.
586                     mStorageManager.verifySpace(mInfo.mDestination, state.mFilename, bytesRead);
587                     forceVerified = true;
588                 } else {
589                     throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
590                             "Failed to write data: " + ex);
591                 }
592             }
593         }
594     }
595
596     /**
597      * Called when we've reached the end of the HTTP response stream, to update the database and
598      * check for consistency.
599      */
600     private void handleEndOfStream(State state) throws StopRequestException {
601         ContentValues values = new ContentValues();
602         values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
603         if (state.mContentLength == -1) {
604             values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, state.mCurrentBytes);
605         }
606         mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
607
608         final boolean lengthMismatched = (state.mContentLength != -1)
609                 && (state.mCurrentBytes != state.mContentLength);
610         if (lengthMismatched) {
611             if (cannotResume(state)) {
612                 throw new StopRequestException(STATUS_CANNOT_RESUME,
613                         "mismatched content length; unable to resume");
614             } else {
615                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
616                         "closed socket before end of file");
617             }
618         }
619     }
620
621     private boolean cannotResume(State state) {
622         return (state.mCurrentBytes > 0 && !mInfo.mNoIntegrity && state.mHeaderETag == null)
623                 || DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType);
624     }
625
626     /**
627      * Read some data from the HTTP response stream, handling I/O errors.
628      * @param data buffer to use to read data
629      * @param entityStream stream for reading the HTTP response entity
630      * @return the number of bytes actually read or -1 if the end of the stream has been reached
631      */
632     private int readFromResponse(State state, byte[] data, InputStream entityStream)
633             throws StopRequestException {
634         try {
635             return entityStream.read(data);
636         } catch (IOException ex) {
637             // TODO: handle stream errors the same as other retries
638             if ("unexpected end of stream".equals(ex.getMessage())) {
639                 return -1;
640             }
641
642             ContentValues values = new ContentValues();
643             values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
644             mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
645             if (cannotResume(state)) {
646                 throw new StopRequestException(STATUS_CANNOT_RESUME,
647                         "Failed reading response: " + ex + "; unable to resume", ex);
648             } else {
649                 throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
650                         "Failed reading response: " + ex, ex);
651             }
652         }
653     }
654
655     /**
656      * Prepare target file based on given network response. Derives filename and
657      * target size as needed.
658      */
659     private void processResponseHeaders(State state, HttpURLConnection conn)
660             throws StopRequestException {
661         // TODO: fallocate the entire file if header gave us specific length
662
663         readResponseHeaders(state, conn);
664
665         state.mFilename = Helpers.generateSaveFile(
666                 mContext,
667                 mInfo.mUri,
668                 mInfo.mHint,
669                 state.mContentDisposition,
670                 state.mContentLocation,
671                 state.mMimeType,
672                 mInfo.mDestination,
673                 state.mContentLength,
674                 mStorageManager);
675
676         updateDatabaseFromHeaders(state);
677         // check connectivity again now that we know the total size
678         checkConnectivity();
679     }
680
681     /**
682      * Update necessary database fields based on values of HTTP response headers that have been
683      * read.
684      */
685     private void updateDatabaseFromHeaders(State state) {
686         ContentValues values = new ContentValues();
687         values.put(Downloads.Impl._DATA, state.mFilename);
688         if (state.mHeaderETag != null) {
689             values.put(Constants.ETAG, state.mHeaderETag);
690         }
691         if (state.mMimeType != null) {
692             values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
693         }
694         values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, mInfo.mTotalBytes);
695         mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
696     }
697
698     /**
699      * Read headers from the HTTP response and store them into local state.
700      */
701     private void readResponseHeaders(State state, HttpURLConnection conn)
702             throws StopRequestException {
703         state.mContentDisposition = conn.getHeaderField("Content-Disposition");
704         state.mContentLocation = conn.getHeaderField("Content-Location");
705
706         if (state.mMimeType == null) {
707             state.mMimeType = Intent.normalizeMimeType(conn.getContentType());
708         }
709
710         state.mHeaderETag = conn.getHeaderField("ETag");
711
712         final String transferEncoding = conn.getHeaderField("Transfer-Encoding");
713         if (transferEncoding == null) {
714             state.mContentLength = getHeaderFieldLong(conn, "Content-Length", -1);
715         } else {
716             Log.i(TAG, "Ignoring Content-Length since Transfer-Encoding is also defined");
717             state.mContentLength = -1;
718         }
719
720         state.mTotalBytes = state.mContentLength;
721         mInfo.mTotalBytes = state.mContentLength;
722
723         final boolean noSizeInfo = state.mContentLength == -1
724                 && (transferEncoding == null || !transferEncoding.equalsIgnoreCase("chunked"));
725         if (!mInfo.mNoIntegrity && noSizeInfo) {
726             throw new StopRequestException(STATUS_CANNOT_RESUME,
727                     "can't know size of download, giving up");
728         }
729     }
730
731     private void parseRetryAfterHeaders(State state, HttpURLConnection conn) {
732         state.mRetryAfter = conn.getHeaderFieldInt("Retry-After", -1);
733         if (state.mRetryAfter < 0) {
734             state.mRetryAfter = 0;
735         } else {
736             if (state.mRetryAfter < Constants.MIN_RETRY_AFTER) {
737                 state.mRetryAfter = Constants.MIN_RETRY_AFTER;
738             } else if (state.mRetryAfter > Constants.MAX_RETRY_AFTER) {
739                 state.mRetryAfter = Constants.MAX_RETRY_AFTER;
740             }
741             state.mRetryAfter += Helpers.sRandom.nextInt(Constants.MIN_RETRY_AFTER + 1);
742             state.mRetryAfter *= 1000;
743         }
744     }
745
746     /**
747      * Prepare the destination file to receive data.  If the file already exists, we'll set up
748      * appropriately for resumption.
749      */
750     private void setupDestinationFile(State state) throws StopRequestException {
751         if (!TextUtils.isEmpty(state.mFilename)) { // only true if we've already run a thread for this download
752             if (Constants.LOGV) {
753                 Log.i(Constants.TAG, "have run thread before for id: " + mInfo.mId +
754                         ", and state.mFilename: " + state.mFilename);
755             }
756             if (!Helpers.isFilenameValid(state.mFilename,
757                     mStorageManager.getDownloadDataDirectory())) {
758                 // this should never happen
759                 throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
760                         "found invalid internal destination filename");
761             }
762             // We're resuming a download that got interrupted
763             File f = new File(state.mFilename);
764             if (f.exists()) {
765                 if (Constants.LOGV) {
766                     Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
767                             ", and state.mFilename: " + state.mFilename);
768                 }
769                 long fileLength = f.length();
770                 if (fileLength == 0) {
771                     // The download hadn't actually started, we can restart from scratch
772                     if (Constants.LOGVV) {
773                         Log.d(TAG, "setupDestinationFile() found fileLength=0, deleting "
774                                 + state.mFilename);
775                     }
776                     f.delete();
777                     state.mFilename = null;
778                     if (Constants.LOGV) {
779                         Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
780                                 ", BUT starting from scratch again: ");
781                     }
782                 } else if (mInfo.mETag == null && !mInfo.mNoIntegrity) {
783                     // This should've been caught upon failure
784                     if (Constants.LOGVV) {
785                         Log.d(TAG, "setupDestinationFile() unable to resume download, deleting "
786                                 + state.mFilename);
787                     }
788                     f.delete();
789                     throw new StopRequestException(Downloads.Impl.STATUS_CANNOT_RESUME,
790                             "Trying to resume a download that can't be resumed");
791                 } else {
792                     // All right, we'll be able to resume this download
793                     if (Constants.LOGV) {
794                         Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
795                                 ", and starting with file of length: " + fileLength);
796                     }
797                     state.mCurrentBytes = (int) fileLength;
798                     if (mInfo.mTotalBytes != -1) {
799                         state.mContentLength = mInfo.mTotalBytes;
800                     }
801                     state.mHeaderETag = mInfo.mETag;
802                     state.mContinuingDownload = true;
803                     if (Constants.LOGV) {
804                         Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
805                                 ", state.mCurrentBytes: " + state.mCurrentBytes +
806                                 ", and setting mContinuingDownload to true: ");
807                     }
808                 }
809             }
810         }
811     }
812
813     /**
814      * Add custom headers for this download to the HTTP request.
815      */
816     private void addRequestHeaders(State state, HttpURLConnection conn) {
817         for (Pair<String, String> header : mInfo.getHeaders()) {
818             conn.addRequestProperty(header.first, header.second);
819         }
820
821         // Only splice in user agent when not already defined
822         if (conn.getRequestProperty("User-Agent") == null) {
823             conn.addRequestProperty("User-Agent", userAgent());
824         }
825
826         // Defeat transparent gzip compression, since it doesn't allow us to
827         // easily resume partial downloads.
828         conn.setRequestProperty("Accept-Encoding", "identity");
829
830         if (state.mContinuingDownload) {
831             if (state.mHeaderETag != null) {
832                 conn.addRequestProperty("If-Match", state.mHeaderETag);
833             }
834             conn.addRequestProperty("Range", "bytes=" + state.mCurrentBytes + "-");
835         }
836     }
837
838     /**
839      * Stores information about the completed download, and notifies the initiating application.
840      */
841     private void notifyDownloadCompleted(
842             State state, int finalStatus, String errorMsg, int numFailed) {
843         notifyThroughDatabase(state, finalStatus, errorMsg, numFailed);
844         if (Downloads.Impl.isStatusCompleted(finalStatus)) {
845             mInfo.sendIntentIfRequested();
846         }
847     }
848
849     private void notifyThroughDatabase(
850             State state, int finalStatus, String errorMsg, int numFailed) {
851         ContentValues values = new ContentValues();
852         values.put(Downloads.Impl.COLUMN_STATUS, finalStatus);
853         values.put(Downloads.Impl._DATA, state.mFilename);
854         values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
855         values.put(Downloads.Impl.COLUMN_LAST_MODIFICATION, mSystemFacade.currentTimeMillis());
856         values.put(Downloads.Impl.COLUMN_FAILED_CONNECTIONS, numFailed);
857         values.put(Constants.RETRY_AFTER_X_REDIRECT_COUNT, state.mRetryAfter);
858
859         if (!TextUtils.equals(mInfo.mUri, state.mRequestUri)) {
860             values.put(Downloads.Impl.COLUMN_URI, state.mRequestUri);
861         }
862
863         // save the error message. could be useful to developers.
864         if (!TextUtils.isEmpty(errorMsg)) {
865             values.put(Downloads.Impl.COLUMN_ERROR_MSG, errorMsg);
866         }
867         mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
868     }
869
870     private INetworkPolicyListener mPolicyListener = new INetworkPolicyListener.Stub() {
871         @Override
872         public void onUidRulesChanged(int uid, int uidRules) {
873             // caller is NPMS, since we only register with them
874             if (uid == mInfo.mUid) {
875                 mPolicyDirty = true;
876             }
877         }
878
879         @Override
880         public void onMeteredIfacesChanged(String[] meteredIfaces) {
881             // caller is NPMS, since we only register with them
882             mPolicyDirty = true;
883         }
884
885         @Override
886         public void onRestrictBackgroundChanged(boolean restrictBackground) {
887             // caller is NPMS, since we only register with them
888             mPolicyDirty = true;
889         }
890     };
891
892     public static long getHeaderFieldLong(URLConnection conn, String field, long defaultValue) {
893         try {
894             return Long.parseLong(conn.getHeaderField(field));
895         } catch (NumberFormatException e) {
896             return defaultValue;
897         }
898     }
899
900     /**
901      * Return if given status is eligible to be treated as
902      * {@link android.provider.Downloads.Impl#STATUS_WAITING_TO_RETRY}.
903      */
904     public static boolean isStatusRetryable(int status) {
905         switch (status) {
906             case STATUS_HTTP_DATA_ERROR:
907             case HTTP_UNAVAILABLE:
908             case HTTP_INTERNAL_ERROR:
909                 return true;
910             default:
911                 return false;
912         }
913     }
914 }