Many improvements to download storage management.
Jeff Sharkey [Thu, 30 Jan 2014 23:01:39 +0000 (15:01 -0800)]
Change all data transfer to occur through FileDescriptors instead of
relying on local files.  This paves the way for downloading directly
to content:// Uris in the future.

Rewrite storage management logic to preflight download when size is
known.  If enough space is found, immediately reserve the space with
fallocate(), advising the kernel block allocator to try giving us a
contiguous block regions to reduce fragmentation.  When preflighting
on internal storage or emulated external storage, ask PackageManager
to clear private app caches to free up space.

Since we fallocate() the entire file, use the database as the source
of truth for resume locations, which requires that we fsync() before
each database update.

Store in-progress downloads in separate directories to keep the OS
from deleting out from under us.  Clean up filename generation logic
to break ties in this new dual-directory case.

Clearer enforcement of successful download preconditions around
content lengths and ETags.  Move all database field mutations to
clearer DownloadInfoDelta object, and write back through single
code path.

Catch and log uncaught exceptions from DownloadThread.  Tests to
verify new storage behaviors.  Fixed existing test to reflect correct
RFC behavior.

Bug: 5287571, 3213677, 12663412
Change-Id: I6bb905eca7c7d1a6bc88df3db28b65d70f660221

18 files changed:
AndroidManifest.xml
src/com/android/providers/downloads/Constants.java
src/com/android/providers/downloads/DownloadInfo.java
src/com/android/providers/downloads/DownloadProvider.java
src/com/android/providers/downloads/DownloadService.java
src/com/android/providers/downloads/DownloadThread.java
src/com/android/providers/downloads/Helpers.java
src/com/android/providers/downloads/StopRequestException.java
src/com/android/providers/downloads/StorageManager.java [deleted file]
src/com/android/providers/downloads/StorageUtils.java [new file with mode: 0644]
tests/AndroidManifest.xml
tests/src/com/android/providers/downloads/AbstractDownloadProviderFunctionalTest.java
tests/src/com/android/providers/downloads/AbstractPublicApiTest.java
tests/src/com/android/providers/downloads/FakeSystemFacade.java
tests/src/com/android/providers/downloads/HelpersTest.java
tests/src/com/android/providers/downloads/PublicApiFunctionalTest.java
tests/src/com/android/providers/downloads/StorageTest.java [new file with mode: 0644]
tests/src/com/android/providers/downloads/ThreadingTest.java

index 031b3d3..423538a 100644 (file)
@@ -54,6 +54,7 @@
     <!-- TODO: replace with READ_NETWORK_POLICY permission when it exists -->
     <uses-permission android:name="android.permission.CONNECTIVITY_INTERNAL" />
     <uses-permission android:name="android.permission.MODIFY_NETWORK_ACCOUNTING" />
+    <uses-permission android:name="android.permission.CLEAR_APP_CACHE" />
 
     <application android:process="android.process.media"
                  android:label="@string/app_label"
index 89210a2..2803d1c 100644 (file)
@@ -80,9 +80,6 @@ public class Constants {
      */
     public static final String FILENAME_SEQUENCE_SEPARATOR = "-";
 
-    /** Where we store downloaded files on the external storage */
-    public static final String DEFAULT_DL_SUBDIR = "/" + Environment.DIRECTORY_DOWNLOADS;
-
     /** A magic filename that is allowed to exist within the system cache */
     public static final String RECOVERY_DIRECTORY = "recovery";
 
@@ -123,16 +120,13 @@ public class Constants {
     public static final String MIMETYPE_APK = "application/vnd.android.package";
 
     /** The buffer size used to stream the data */
-    public static final int BUFFER_SIZE = 4096;
+    public static final int BUFFER_SIZE = 8192;
 
     /** The minimum amount of progress that has to be done before the progress bar gets updated */
-    public static final int MIN_PROGRESS_STEP = 4096;
+    public static final int MIN_PROGRESS_STEP = 65536;
 
     /** The minimum amount of time that has to elapse before the progress bar gets updated, in ms */
-    public static final long MIN_PROGRESS_TIME = 1500;
-
-    /** The maximum number of rows in the database (FIFO) */
-    public static final int MAX_DOWNLOADS = 1000;
+    public static final long MIN_PROGRESS_TIME = 2000;
 
     /**
      * The number of times that the download manager will retry its network
@@ -177,4 +171,9 @@ public class Constants {
 
     public static final String STORAGE_AUTHORITY = "com.android.providers.downloads.documents";
     public static final String STORAGE_ROOT_ID = "downloads";
+
+    /**
+     * Name of directory on cache partition containing in-progress downloads.
+     */
+    public static final String DIRECTORY_CACHE_RUNNING = "partial_downloads";
 }
index 7a912d5..3571a78 100644 (file)
@@ -36,6 +36,7 @@ import android.util.Pair;
 import com.android.internal.annotations.GuardedBy;
 import com.android.internal.util.IndentingPrintWriter;
 
+import java.io.CharArrayWriter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -45,7 +46,8 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
 /**
- * Stores information about an individual download.
+ * Details about a specific download. Fields should only be mutated by updating
+ * from database query.
  */
 public class DownloadInfo {
     // TODO: move towards these in-memory objects being sources of truth, and
@@ -60,10 +62,9 @@ public class DownloadInfo {
             mCursor = cursor;
         }
 
-        public DownloadInfo newDownloadInfo(Context context, SystemFacade systemFacade,
-                StorageManager storageManager, DownloadNotifier notifier) {
-            final DownloadInfo info = new DownloadInfo(
-                    context, systemFacade, storageManager, notifier);
+        public DownloadInfo newDownloadInfo(
+                Context context, SystemFacade systemFacade, DownloadNotifier notifier) {
+            final DownloadInfo info = new DownloadInfo(context, systemFacade, notifier);
             updateFromDatabase(info);
             readRequestHeaders(info);
             return info;
@@ -75,7 +76,7 @@ public class DownloadInfo {
             info.mNoIntegrity = getInt(Downloads.Impl.COLUMN_NO_INTEGRITY) == 1;
             info.mHint = getString(Downloads.Impl.COLUMN_FILE_NAME_HINT);
             info.mFileName = getString(Downloads.Impl._DATA);
-            info.mMimeType = getString(Downloads.Impl.COLUMN_MIME_TYPE);
+            info.mMimeType = Intent.normalizeMimeType(getString(Downloads.Impl.COLUMN_MIME_TYPE));
             info.mDestination = getInt(Downloads.Impl.COLUMN_DESTINATION);
             info.mVisibility = getInt(Downloads.Impl.COLUMN_VISIBILITY);
             info.mStatus = getInt(Downloads.Impl.COLUMN_STATUS);
@@ -206,6 +207,7 @@ public class DownloadInfo {
 
     public long mId;
     public String mUri;
+    @Deprecated
     public boolean mNoIntegrity;
     public String mHint;
     public String mFileName;
@@ -254,14 +256,11 @@ public class DownloadInfo {
 
     private final Context mContext;
     private final SystemFacade mSystemFacade;
-    private final StorageManager mStorageManager;
     private final DownloadNotifier mNotifier;
 
-    private DownloadInfo(Context context, SystemFacade systemFacade, StorageManager storageManager,
-            DownloadNotifier notifier) {
+    private DownloadInfo(Context context, SystemFacade systemFacade, DownloadNotifier notifier) {
         mContext = context;
         mSystemFacade = systemFacade;
-        mStorageManager = storageManager;
         mNotifier = notifier;
         mFuzz = Helpers.sRandom.nextInt(1001);
     }
@@ -270,6 +269,14 @@ public class DownloadInfo {
         return Collections.unmodifiableList(mRequestHeaders);
     }
 
+    public String getUserAgent() {
+        if (mUserAgent != null) {
+            return mUserAgent;
+        } else {
+            return Constants.DEFAULT_USER_AGENT;
+        }
+    }
+
     public void sendIntentIfRequested() {
         if (mPackage == null) {
             return;
@@ -329,7 +336,7 @@ public class DownloadInfo {
 
             case Downloads.Impl.STATUS_WAITING_FOR_NETWORK:
             case Downloads.Impl.STATUS_QUEUED_FOR_WIFI:
-                return checkCanUseNetwork() == NetworkState.OK;
+                return checkCanUseNetwork(mTotalBytes) == NetworkState.OK;
 
             case Downloads.Impl.STATUS_WAITING_TO_RETRY:
                 // download was waiting for a delayed restart
@@ -362,7 +369,7 @@ public class DownloadInfo {
     /**
      * Returns whether this download is allowed to use the network.
      */
-    public NetworkState checkCanUseNetwork() {
+    public NetworkState checkCanUseNetwork(long totalBytes) {
         final NetworkInfo info = mSystemFacade.getActiveNetworkInfo(mUid);
         if (info == null || !info.isConnected()) {
             return NetworkState.NO_CONNECTION;
@@ -376,7 +383,7 @@ public class DownloadInfo {
         if (mSystemFacade.isActiveNetworkMetered() && !mAllowMetered) {
             return NetworkState.TYPE_DISALLOWED_BY_REQUESTOR;
         }
-        return checkIsNetworkTypeAllowed(info.getType());
+        return checkIsNetworkTypeAllowed(info.getType(), totalBytes);
     }
 
     private boolean isRoamingAllowed() {
@@ -392,7 +399,7 @@ public class DownloadInfo {
      * @param networkType a constant from ConnectivityManager.TYPE_*.
      * @return one of the NETWORK_* constants
      */
-    private NetworkState checkIsNetworkTypeAllowed(int networkType) {
+    private NetworkState checkIsNetworkTypeAllowed(int networkType, long totalBytes) {
         if (mIsPublicApi) {
             final int flag = translateNetworkTypeToApiFlag(networkType);
             final boolean allowAllNetworkTypes = mAllowedNetworkTypes == ~0;
@@ -400,7 +407,7 @@ public class DownloadInfo {
                 return NetworkState.TYPE_DISALLOWED_BY_REQUESTOR;
             }
         }
-        return checkSizeAllowedForNetwork(networkType);
+        return checkSizeAllowedForNetwork(networkType, totalBytes);
     }
 
     /**
@@ -427,24 +434,27 @@ public class DownloadInfo {
      * Check if the download's size prohibits it from running over the current network.
      * @return one of the NETWORK_* constants
      */
-    private NetworkState checkSizeAllowedForNetwork(int networkType) {
-        if (mTotalBytes <= 0) {
-            return NetworkState.OK; // we don't know the size yet
-        }
-        if (networkType == ConnectivityManager.TYPE_WIFI) {
-            return NetworkState.OK; // anything goes over wifi
-        }
-        Long maxBytesOverMobile = mSystemFacade.getMaxBytesOverMobile();
-        if (maxBytesOverMobile != null && mTotalBytes > maxBytesOverMobile) {
-            return NetworkState.UNUSABLE_DUE_TO_SIZE;
-        }
-        if (mBypassRecommendedSizeLimit == 0) {
-            Long recommendedMaxBytesOverMobile = mSystemFacade.getRecommendedMaxBytesOverMobile();
-            if (recommendedMaxBytesOverMobile != null
-                    && mTotalBytes > recommendedMaxBytesOverMobile) {
-                return NetworkState.RECOMMENDED_UNUSABLE_DUE_TO_SIZE;
+    private NetworkState checkSizeAllowedForNetwork(int networkType, long totalBytes) {
+        if (totalBytes <= 0) {
+            // we don't know the size yet
+            return NetworkState.OK;
+        }
+
+        if (ConnectivityManager.isNetworkTypeMobile(networkType)) {
+            Long maxBytesOverMobile = mSystemFacade.getMaxBytesOverMobile();
+            if (maxBytesOverMobile != null && totalBytes > maxBytesOverMobile) {
+                return NetworkState.UNUSABLE_DUE_TO_SIZE;
+            }
+            if (mBypassRecommendedSizeLimit == 0) {
+                Long recommendedMaxBytesOverMobile = mSystemFacade
+                        .getRecommendedMaxBytesOverMobile();
+                if (recommendedMaxBytesOverMobile != null
+                        && totalBytes > recommendedMaxBytesOverMobile) {
+                    return NetworkState.RECOMMENDED_UNUSABLE_DUE_TO_SIZE;
+                }
             }
         }
+
         return NetworkState.OK;
     }
 
@@ -467,8 +477,7 @@ public class DownloadInfo {
                     mContext.getContentResolver().update(getAllDownloadsUri(), values, null, null);
                 }
 
-                mTask = new DownloadThread(
-                        mContext, mSystemFacade, this, mStorageManager, mNotifier);
+                mTask = new DownloadThread(mContext, mSystemFacade, mNotifier, this);
                 mSubmittedTask = executor.submit(mTask);
             }
             return isReady;
@@ -506,6 +515,13 @@ public class DownloadInfo {
         return ContentUris.withAppendedId(Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI, mId);
     }
 
+    @Override
+    public String toString() {
+        final CharArrayWriter writer = new CharArrayWriter();
+        dump(new IndentingPrintWriter(writer, "  "));
+        return writer.toString();
+    }
+
     public void dump(IndentingPrintWriter pw) {
         pw.println("DownloadInfo:");
         pw.increaseIndent();
index ad3cf7a..dc3c480 100644 (file)
@@ -178,7 +178,6 @@ public final class DownloadProvider extends ContentProvider {
     /** List of uids that can access the downloads */
     private int mSystemUid = -1;
     private int mDefContainerUid = -1;
-    private File mDownloadsDataDir;
 
     @VisibleForTesting
     SystemFacade mSystemFacade;
@@ -464,9 +463,8 @@ public final class DownloadProvider extends ContentProvider {
         // saves us by getting some initialization code in DownloadService out of the way.
         Context context = getContext();
         context.startService(new Intent(context, DownloadService.class));
-        mDownloadsDataDir = StorageManager.getDownloadDataDirectory(getContext());
         try {
-            SELinux.restorecon(mDownloadsDataDir.getCanonicalPath());
+            SELinux.restorecon(context.getCacheDir().getCanonicalPath());
         } catch (IOException e) {
             Log.wtf(Constants.TAG, "Could not get canonical path for download directory", e);
         }
@@ -540,7 +538,7 @@ public final class DownloadProvider extends ContentProvider {
         // validate the destination column
         Integer dest = values.getAsInteger(Downloads.Impl.COLUMN_DESTINATION);
         if (dest != null) {
-            if (getContext().checkCallingPermission(Downloads.Impl.PERMISSION_ACCESS_ADVANCED)
+            if (getContext().checkCallingOrSelfPermission(Downloads.Impl.PERMISSION_ACCESS_ADVANCED)
                     != PackageManager.PERMISSION_GRANTED
                     && (dest == Downloads.Impl.DESTINATION_CACHE_PARTITION
                             || dest == Downloads.Impl.DESTINATION_CACHE_PARTITION_NOROAMING
@@ -551,7 +549,7 @@ public final class DownloadProvider extends ContentProvider {
             // for public API behavior, if an app has CACHE_NON_PURGEABLE permission, automatically
             // switch to non-purgeable download
             boolean hasNonPurgeablePermission =
-                    getContext().checkCallingPermission(
+                    getContext().checkCallingOrSelfPermission(
                             Downloads.Impl.PERMISSION_CACHE_NON_PURGEABLE)
                             == PackageManager.PERMISSION_GRANTED;
             if (isPublicApi && dest == Downloads.Impl.DESTINATION_CACHE_PARTITION_PURGEABLE
@@ -638,7 +636,7 @@ public final class DownloadProvider extends ContentProvider {
         copyString(Downloads.Impl.COLUMN_REFERER, values, filteredValues);
 
         // UID, PID columns
-        if (getContext().checkCallingPermission(Downloads.Impl.PERMISSION_ACCESS_ADVANCED)
+        if (getContext().checkCallingOrSelfPermission(Downloads.Impl.PERMISSION_ACCESS_ADVANCED)
                 == PackageManager.PERMISSION_GRANTED) {
             copyInteger(Downloads.Impl.COLUMN_OTHER_UID, values, filteredValues);
         }
@@ -1123,7 +1121,7 @@ public final class DownloadProvider extends ContentProvider {
             selection.appendClause(Downloads.Impl._ID + " = ?", getDownloadIdFromUri(uri));
         }
         if ((uriMatch == MY_DOWNLOADS || uriMatch == MY_DOWNLOADS_ID)
-                && getContext().checkCallingPermission(Downloads.Impl.PERMISSION_ACCESS_ALL)
+                && getContext().checkCallingOrSelfPermission(Downloads.Impl.PERMISSION_ACCESS_ALL)
                 != PackageManager.PERMISSION_GRANTED) {
             selection.appendClause(
                     Constants.UID + "= ? OR " + Downloads.Impl.COLUMN_OTHER_UID + "= ?",
@@ -1205,11 +1203,12 @@ public final class DownloadProvider extends ContentProvider {
         if (path == null) {
             throw new FileNotFoundException("No filename found.");
         }
-        if (!Helpers.isFilenameValid(path, mDownloadsDataDir)) {
-            throw new FileNotFoundException("Invalid filename: " + path);
-        }
 
         final File file = new File(path);
+        if (!Helpers.isFilenameValid(getContext(), file)) {
+            throw new FileNotFoundException("Invalid file: " + file);
+        }
+
         if ("r".equals(mode)) {
             return ParcelFileDescriptor.open(file, ParcelFileDescriptor.MODE_READ_ONLY);
         } else {
index 7d746cc..084a359 100644 (file)
@@ -54,7 +54,10 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
@@ -78,7 +81,6 @@ public class DownloadService extends Service {
     SystemFacade mSystemFacade;
 
     private AlarmManager mAlarmManager;
-    private StorageManager mStorageManager;
 
     /** Observer to get notified when the content observer's data changes */
     private DownloadManagerContentObserver mObserver;
@@ -105,7 +107,28 @@ public class DownloadService extends Service {
         // threads as needed (up to maximum) and reclaims them when finished.
         final ThreadPoolExecutor executor = new ThreadPoolExecutor(
                 maxConcurrent, maxConcurrent, 10, TimeUnit.SECONDS,
-                new LinkedBlockingQueue<Runnable>());
+                new LinkedBlockingQueue<Runnable>()) {
+            @Override
+            protected void afterExecute(Runnable r, Throwable t) {
+                super.afterExecute(r, t);
+
+                if (t == null && r instanceof Future<?>) {
+                    try {
+                        ((Future<?>) r).get();
+                    } catch (CancellationException ce) {
+                        t = ce;
+                    } catch (ExecutionException ee) {
+                        t = ee.getCause();
+                    } catch (InterruptedException ie) {
+                        Thread.currentThread().interrupt();
+                    }
+                }
+
+                if (t != null) {
+                    Log.w(TAG, "Uncaught exception", t);
+                }
+            }
+        };
         executor.allowCoreThreadTimeOut(true);
         return executor;
     }
@@ -157,7 +180,6 @@ public class DownloadService extends Service {
         }
 
         mAlarmManager = (AlarmManager) getSystemService(Context.ALARM_SERVICE);
-        mStorageManager = new StorageManager(this);
 
         mUpdateThread = new HandlerThread(TAG + "-UpdateThread");
         mUpdateThread.start();
@@ -198,9 +220,11 @@ public class DownloadService extends Service {
     /**
      * Enqueue an {@link #updateLocked()} pass to occur in future.
      */
-    private void enqueueUpdate() {
-        mUpdateHandler.removeMessages(MSG_UPDATE);
-        mUpdateHandler.obtainMessage(MSG_UPDATE, mLastStartId, -1).sendToTarget();
+    public void enqueueUpdate() {
+        if (mUpdateHandler != null) {
+            mUpdateHandler.removeMessages(MSG_UPDATE);
+            mUpdateHandler.obtainMessage(MSG_UPDATE, mLastStartId, -1).sendToTarget();
+        }
     }
 
     /**
@@ -376,8 +400,7 @@ public class DownloadService extends Service {
      * download if appropriate.
      */
     private DownloadInfo insertDownloadLocked(DownloadInfo.Reader reader, long now) {
-        final DownloadInfo info = reader.newDownloadInfo(
-                this, mSystemFacade, mStorageManager, mNotifier);
+        final DownloadInfo info = reader.newDownloadInfo(this, mSystemFacade, mNotifier);
         mDownloads.put(info.mId, info);
 
         if (Constants.LOGVV) {
index 93f8d65..fd4e89a 100644 (file)
@@ -22,6 +22,8 @@ import static android.provider.Downloads.Impl.STATUS_FILE_ERROR;
 import static android.provider.Downloads.Impl.STATUS_HTTP_DATA_ERROR;
 import static android.provider.Downloads.Impl.STATUS_SUCCESS;
 import static android.provider.Downloads.Impl.STATUS_TOO_MANY_REDIRECTS;
+import static android.provider.Downloads.Impl.STATUS_UNHANDLED_HTTP_CODE;
+import static android.provider.Downloads.Impl.STATUS_UNKNOWN_ERROR;
 import static android.provider.Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
 import static android.provider.Downloads.Impl.STATUS_WAITING_TO_RETRY;
 import static android.text.format.DateUtils.SECOND_IN_MILLIS;
@@ -31,8 +33,10 @@ import static java.net.HttpURLConnection.HTTP_MOVED_PERM;
 import static java.net.HttpURLConnection.HTTP_MOVED_TEMP;
 import static java.net.HttpURLConnection.HTTP_OK;
 import static java.net.HttpURLConnection.HTTP_PARTIAL;
+import static java.net.HttpURLConnection.HTTP_PRECON_FAILED;
 import static java.net.HttpURLConnection.HTTP_SEE_OTHER;
 import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
+import static libcore.io.OsConstants.SEEK_SET;
 
 import android.content.ContentValues;
 import android.content.Context;
@@ -44,148 +48,171 @@ import android.net.INetworkPolicyListener;
 import android.net.NetworkInfo;
 import android.net.NetworkPolicyManager;
 import android.net.TrafficStats;
-import android.os.FileUtils;
+import android.os.ParcelFileDescriptor;
 import android.os.PowerManager;
 import android.os.Process;
 import android.os.SystemClock;
 import android.os.WorkSource;
 import android.provider.Downloads;
-import android.text.TextUtils;
 import android.util.Log;
 import android.util.Pair;
 
 import com.android.providers.downloads.DownloadInfo.NetworkState;
 
+import libcore.io.ErrnoException;
 import libcore.io.IoUtils;
+import libcore.io.Libcore;
+import libcore.io.OsConstants;
 
 import java.io.File;
 import java.io.FileDescriptor;
-import java.io.FileOutputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.io.RandomAccessFile;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
+import java.net.ProtocolException;
 import java.net.URL;
 import java.net.URLConnection;
 
 /**
  * Task which executes a given {@link DownloadInfo}: making network requests,
  * persisting data to disk, and updating {@link DownloadProvider}.
+ * <p>
+ * To know if a download is successful, we need to know either the final content
+ * length to expect, or the transfer to be chunked. To resume an interrupted
+ * download, we need an ETag.
+ * <p>
+ * Failed network requests are retried several times before giving up. Local
+ * disk errors fail immediately and are not retried.
  */
 public class DownloadThread implements Runnable {
 
     // TODO: bind each download to a specific network interface to avoid state
     // checking races once we have ConnectivityManager API
 
+    // TODO: add support for saving to content://
+
     private static final int HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416;
     private static final int HTTP_TEMP_REDIRECT = 307;
 
     private static final int DEFAULT_TIMEOUT = (int) (20 * SECOND_IN_MILLIS);
 
     private final Context mContext;
-    private final DownloadInfo mInfo;
     private final SystemFacade mSystemFacade;
-    private final StorageManager mStorageManager;
     private final DownloadNotifier mNotifier;
 
-    private volatile boolean mPolicyDirty;
-
-    public DownloadThread(Context context, SystemFacade systemFacade, DownloadInfo info,
-            StorageManager storageManager, DownloadNotifier notifier) {
-        mContext = context;
-        mSystemFacade = systemFacade;
-        mInfo = info;
-        mStorageManager = storageManager;
-        mNotifier = notifier;
-    }
+    private final long mId;
 
     /**
-     * Returns the user agent provided by the initiating app, or use the default one
+     * Info object that should be treated as read-only. Any potentially mutated
+     * fields are tracked in {@link #mInfoDelta}. If a field exists in
+     * {@link #mInfoDelta}, it must not be read from {@link #mInfo}.
      */
-    private String userAgent() {
-        String userAgent = mInfo.mUserAgent;
-        if (userAgent == null) {
-            userAgent = Constants.DEFAULT_USER_AGENT;
-        }
-        return userAgent;
-    }
+    private final DownloadInfo mInfo;
+    private final DownloadInfoDelta mInfoDelta;
+
+    private volatile boolean mPolicyDirty;
 
     /**
-     * State for the entire run() method.
+     * Local changes to {@link DownloadInfo}. These are kept local to avoid
+     * racing with the thread that updates based on change notifications.
      */
-    static class State {
-        public String mFilename;
+    private class DownloadInfoDelta {
+        public String mUri;
+        public String mFileName;
         public String mMimeType;
-        public int mRetryAfter = 0;
-        public boolean mGotData = false;
-        public String mRequestUri;
-        public long mTotalBytes = -1;
-        public long mCurrentBytes = 0;
-        public String mHeaderETag;
-        public boolean mContinuingDownload = false;
-        public long mBytesNotified = 0;
-        public long mTimeLastNotification = 0;
-        public int mNetworkType = ConnectivityManager.TYPE_NONE;
-
-        /** Historical bytes/second speed of this download. */
-        public long mSpeed;
-        /** Time when current sample started. */
-        public long mSpeedSampleStart;
-        /** Bytes transferred since current sample started. */
-        public long mSpeedSampleBytes;
-
-        public long mContentLength = -1;
-        public String mContentDisposition;
-        public String mContentLocation;
-
-        public int mRedirectionCount;
-        public URL mUrl;
-
-        public State(DownloadInfo info) {
-            mMimeType = Intent.normalizeMimeType(info.mMimeType);
-            mRequestUri = info.mUri;
-            mFilename = info.mFileName;
+        public int mStatus;
+        public int mNumFailed;
+        public int mRetryAfter;
+        public long mTotalBytes;
+        public long mCurrentBytes;
+        public String mETag;
+
+        public String mErrorMsg;
+
+        public DownloadInfoDelta(DownloadInfo info) {
+            mUri = info.mUri;
+            mFileName = info.mFileName;
+            mMimeType = info.mMimeType;
+            mStatus = info.mStatus;
+            mNumFailed = info.mNumFailed;
+            mRetryAfter = info.mRetryAfter;
             mTotalBytes = info.mTotalBytes;
             mCurrentBytes = info.mCurrentBytes;
+            mETag = info.mETag;
         }
 
-        public void resetBeforeExecute() {
-            // Reset any state from previous execution
-            mContentLength = -1;
-            mContentDisposition = null;
-            mContentLocation = null;
-            mRedirectionCount = 0;
+        /**
+         * Push update of current delta values to provider.
+         */
+        public void writeToDatabase() {
+            final ContentValues values = new ContentValues();
+
+            values.put(Downloads.Impl.COLUMN_URI, mUri);
+            values.put(Downloads.Impl._DATA, mFileName);
+            values.put(Downloads.Impl.COLUMN_MIME_TYPE, mMimeType);
+            values.put(Downloads.Impl.COLUMN_STATUS, mStatus);
+            values.put(Downloads.Impl.COLUMN_FAILED_CONNECTIONS, mNumFailed);
+            values.put(Constants.RETRY_AFTER_X_REDIRECT_COUNT, mRetryAfter);
+            values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, mTotalBytes);
+            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, mCurrentBytes);
+            values.put(Constants.ETAG, mETag);
+
+            values.put(Downloads.Impl.COLUMN_LAST_MODIFICATION, mSystemFacade.currentTimeMillis());
+            values.put(Downloads.Impl.COLUMN_ERROR_MSG, mErrorMsg);
+
+            mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
         }
     }
 
+    /**
+     * Flag indicating if we've made forward progress transferring file data
+     * from a remote server.
+     */
+    private boolean mMadeProgress = false;
+
+    /**
+     * Details from the last time we pushed a database update.
+     */
+    private long mLastUpdateBytes = 0;
+    private long mLastUpdateTime = 0;
+
+    private int mNetworkType = ConnectivityManager.TYPE_NONE;
+
+    /** Historical bytes/second speed of this download. */
+    private long mSpeed;
+    /** Time when current sample started. */
+    private long mSpeedSampleStart;
+    /** Bytes transferred since current sample started. */
+    private long mSpeedSampleBytes;
+
+    public DownloadThread(Context context, SystemFacade systemFacade, DownloadNotifier notifier,
+            DownloadInfo info) {
+        mContext = context;
+        mSystemFacade = systemFacade;
+        mNotifier = notifier;
+
+        mId = info.mId;
+        mInfo = info;
+        mInfoDelta = new DownloadInfoDelta(info);
+    }
+
     @Override
     public void run() {
         Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND);
-        try {
-            runInternal();
-        } finally {
-            mNotifier.notifyDownloadSpeed(mInfo.mId, 0);
-        }
-    }
 
-    private void runInternal() {
         // Skip when download already marked as finished; this download was
         // probably started again while racing with UpdateThread.
-        if (DownloadInfo.queryDownloadStatus(mContext.getContentResolver(), mInfo.mId)
+        if (DownloadInfo.queryDownloadStatus(mContext.getContentResolver(), mId)
                 == Downloads.Impl.STATUS_SUCCESS) {
-            Log.d(TAG, "Download " + mInfo.mId + " already finished; skipping");
+            logDebug("Already finished; skipping");
             return;
         }
 
-        State state = new State(mInfo);
-        PowerManager.WakeLock wakeLock = null;
-        int finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
-        int numFailed = mInfo.mNumFailed;
-        String errorMsg = null;
-
         final NetworkPolicyManager netPolicy = NetworkPolicyManager.from(mContext);
+        PowerManager.WakeLock wakeLock = null;
         final PowerManager pm = (PowerManager) mContext.getSystemService(Context.POWER_SERVICE);
 
         try {
@@ -196,13 +223,13 @@ public class DownloadThread implements Runnable {
             // while performing download, register for rules updates
             netPolicy.registerListener(mPolicyListener);
 
-            Log.i(Constants.TAG, "Download " + mInfo.mId + " starting");
+            logDebug("Starting");
 
             // Remember which network this download started on; used to
             // determine if errors were due to network changes.
             final NetworkInfo info = mSystemFacade.getActiveNetworkInfo(mInfo.mUid);
             if (info != null) {
-                state.mNetworkType = info.getType();
+                mNetworkType = info.getType();
             }
 
             // Network traffic on this thread should be counted against the
@@ -210,75 +237,79 @@ public class DownloadThread implements Runnable {
             TrafficStats.setThreadStatsTag(TrafficStats.TAG_SYSTEM_DOWNLOAD);
             TrafficStats.setThreadStatsUid(mInfo.mUid);
 
-            try {
-                // TODO: migrate URL sanity checking into client side of API
-                state.mUrl = new URL(state.mRequestUri);
-            } catch (MalformedURLException e) {
-                throw new StopRequestException(STATUS_BAD_REQUEST, e);
-            }
+            executeDownload();
+
+            mInfoDelta.mStatus = STATUS_SUCCESS;
+            TrafficStats.incrementOperationCount(1);
 
-            executeDownload(state);
-
-            finalizeDestinationFile(state);
-            finalStatus = Downloads.Impl.STATUS_SUCCESS;
-        } catch (StopRequestException error) {
-            // remove the cause before printing, in case it contains PII
-            errorMsg = error.getMessage();
-            String msg = "Aborting request for download " + mInfo.mId + ": " + errorMsg;
-            Log.w(Constants.TAG, msg);
-            if (Constants.LOGV) {
-                Log.w(Constants.TAG, msg, error);
+            // If we just finished a chunked file, record total size
+            if (mInfoDelta.mTotalBytes == -1) {
+                mInfoDelta.mTotalBytes = mInfoDelta.mCurrentBytes;
             }
-            finalStatus = error.getFinalStatus();
+
+        } catch (StopRequestException e) {
+            mInfoDelta.mStatus = e.getFinalStatus();
+            mInfoDelta.mErrorMsg = e.getMessage();
+
+            logWarning("Stop requested with status "
+                    + Downloads.Impl.statusToString(mInfoDelta.mStatus) + ": "
+                    + mInfoDelta.mErrorMsg);
 
             // Nobody below our level should request retries, since we handle
             // failure counts at this level.
-            if (finalStatus == STATUS_WAITING_TO_RETRY) {
+            if (mInfoDelta.mStatus == STATUS_WAITING_TO_RETRY) {
                 throw new IllegalStateException("Execution should always throw final error codes");
             }
 
             // Some errors should be retryable, unless we fail too many times.
-            if (isStatusRetryable(finalStatus)) {
-                if (state.mGotData) {
-                    numFailed = 1;
+            if (isStatusRetryable(mInfoDelta.mStatus)) {
+                if (mMadeProgress) {
+                    mInfoDelta.mNumFailed = 1;
                 } else {
-                    numFailed += 1;
+                    mInfoDelta.mNumFailed += 1;
                 }
 
-                if (numFailed < Constants.MAX_RETRIES) {
+                if (mInfoDelta.mNumFailed < Constants.MAX_RETRIES) {
                     final NetworkInfo info = mSystemFacade.getActiveNetworkInfo(mInfo.mUid);
-                    if (info != null && info.getType() == state.mNetworkType
-                            && info.isConnected()) {
+                    if (info != null && info.getType() == mNetworkType && info.isConnected()) {
                         // Underlying network is still intact, use normal backoff
-                        finalStatus = STATUS_WAITING_TO_RETRY;
+                        mInfoDelta.mStatus = STATUS_WAITING_TO_RETRY;
                     } else {
                         // Network changed, retry on any next available
-                        finalStatus = STATUS_WAITING_FOR_NETWORK;
+                        mInfoDelta.mStatus = STATUS_WAITING_FOR_NETWORK;
+                    }
+
+                    if ((mInfoDelta.mETag == null && mMadeProgress)
+                            || DownloadDrmHelper.isDrmConvertNeeded(mInfoDelta.mMimeType)) {
+                        // However, if we wrote data and have no ETag to verify
+                        // contents against later, we can't actually resume.
+                        mInfoDelta.mStatus = STATUS_CANNOT_RESUME;
                     }
                 }
             }
 
-            // fall through to finally block
-        } catch (Throwable ex) {
-            errorMsg = ex.getMessage();
-            String msg = "Exception for id " + mInfo.mId + ": " + errorMsg;
-            Log.w(Constants.TAG, msg, ex);
-            finalStatus = Downloads.Impl.STATUS_UNKNOWN_ERROR;
-            // falls through to the code that reports an error
+        } catch (Throwable t) {
+            mInfoDelta.mStatus = STATUS_UNKNOWN_ERROR;
+            mInfoDelta.mErrorMsg = t.toString();
+
+            logError("Failed: " + mInfoDelta.mErrorMsg, t);
+
         } finally {
-            if (finalStatus == STATUS_SUCCESS) {
-                TrafficStats.incrementOperationCount(1);
+            logDebug("Finished with status " + Downloads.Impl.statusToString(mInfoDelta.mStatus));
+
+            mNotifier.notifyDownloadSpeed(mId, 0);
+
+            finalizeDestination();
+
+            mInfoDelta.writeToDatabase();
+
+            if (Downloads.Impl.isStatusCompleted(mInfoDelta.mStatus)) {
+                mInfo.sendIntentIfRequested();
             }
 
             TrafficStats.clearThreadStatsTag();
             TrafficStats.clearThreadStatsUid();
 
-            cleanupDestination(state, finalStatus);
-            notifyDownloadCompleted(state, finalStatus, errorMsg, numFailed);
-
-            Log.i(Constants.TAG, "Download " + mInfo.mId + " finished with status "
-                    + Downloads.Impl.statusToString(finalStatus));
-
             netPolicy.unregisterListener(mPolicyListener);
 
             if (wakeLock != null) {
@@ -286,54 +317,54 @@ public class DownloadThread implements Runnable {
                 wakeLock = null;
             }
         }
-        mStorageManager.incrementNumDownloadsSoFar();
     }
 
     /**
      * Fully execute a single download request. Setup and send the request,
      * handle the response, and transfer the data to the destination file.
      */
-    private void executeDownload(State state) throws StopRequestException {
-        state.resetBeforeExecute();
-        setupDestinationFile(state);
-
-        // skip when already finished; remove after fixing race in 5217390
-        if (state.mCurrentBytes == state.mTotalBytes) {
-            Log.i(Constants.TAG, "Skipping initiating request for download " +
-                  mInfo.mId + "; already completed");
-            return;
+    private void executeDownload() throws StopRequestException {
+        final boolean resuming = mInfoDelta.mCurrentBytes != 0;
+
+        URL url;
+        try {
+            // TODO: migrate URL sanity checking into client side of API
+            url = new URL(mInfoDelta.mUri);
+        } catch (MalformedURLException e) {
+            throw new StopRequestException(STATUS_BAD_REQUEST, e);
         }
 
-        while (state.mRedirectionCount++ < Constants.MAX_REDIRECTS) {
+        int redirectionCount = 0;
+        while (redirectionCount++ < Constants.MAX_REDIRECTS) {
             // Open connection and follow any redirects until we have a useful
             // response with body.
             HttpURLConnection conn = null;
             try {
                 checkConnectivity();
-                conn = (HttpURLConnection) state.mUrl.openConnection();
+                conn = (HttpURLConnection) url.openConnection();
                 conn.setInstanceFollowRedirects(false);
                 conn.setConnectTimeout(DEFAULT_TIMEOUT);
                 conn.setReadTimeout(DEFAULT_TIMEOUT);
 
-                addRequestHeaders(state, conn);
+                addRequestHeaders(conn, resuming);
 
                 final int responseCode = conn.getResponseCode();
                 switch (responseCode) {
                     case HTTP_OK:
-                        if (state.mContinuingDownload) {
+                        if (resuming) {
                             throw new StopRequestException(
                                     STATUS_CANNOT_RESUME, "Expected partial, but received OK");
                         }
-                        processResponseHeaders(state, conn);
-                        transferData(state, conn);
+                        parseOkHeaders(conn);
+                        transferData(conn);
                         return;
 
                     case HTTP_PARTIAL:
-                        if (!state.mContinuingDownload) {
+                        if (!resuming) {
                             throw new StopRequestException(
                                     STATUS_CANNOT_RESUME, "Expected OK, but received partial");
                         }
-                        transferData(state, conn);
+                        transferData(conn);
                         return;
 
                     case HTTP_MOVED_PERM:
@@ -341,19 +372,23 @@ public class DownloadThread implements Runnable {
                     case HTTP_SEE_OTHER:
                     case HTTP_TEMP_REDIRECT:
                         final String location = conn.getHeaderField("Location");
-                        state.mUrl = new URL(state.mUrl, location);
+                        url = new URL(url, location);
                         if (responseCode == HTTP_MOVED_PERM) {
                             // Push updated URL back to database
-                            state.mRequestUri = state.mUrl.toString();
+                            mInfoDelta.mUri = url.toString();
                         }
                         continue;
 
+                    case HTTP_PRECON_FAILED:
+                        throw new StopRequestException(
+                                STATUS_CANNOT_RESUME, "Precondition failed");
+
                     case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE:
                         throw new StopRequestException(
                                 STATUS_CANNOT_RESUME, "Requested range not satisfiable");
 
                     case HTTP_UNAVAILABLE:
-                        parseRetryAfterHeaders(state, conn);
+                        parseUnavailableHeaders(conn);
                         throw new StopRequestException(
                                 HTTP_UNAVAILABLE, conn.getResponseMessage());
 
@@ -365,9 +400,15 @@ public class DownloadThread implements Runnable {
                         StopRequestException.throwUnhandledHttpError(
                                 responseCode, conn.getResponseMessage());
                 }
+
             } catch (IOException e) {
-                // Trouble with low-level sockets
-                throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
+                if (e instanceof ProtocolException
+                        && e.getMessage().startsWith("Unexpected status line")) {
+                    throw new StopRequestException(STATUS_UNHANDLED_HTTP_CODE, e);
+                } else {
+                    // Trouble with low-level sockets
+                    throw new StopRequestException(STATUS_HTTP_DATA_ERROR, e);
+                }
 
             } finally {
                 if (conn != null) conn.disconnect();
@@ -380,11 +421,23 @@ public class DownloadThread implements Runnable {
     /**
      * Transfer data from the given connection to the destination file.
      */
-    private void transferData(State state, HttpURLConnection conn) throws StopRequestException {
+    private void transferData(HttpURLConnection conn) throws StopRequestException {
+
+        // To detect when we're really finished, we either need a length or
+        // chunked encoding.
+        final boolean hasLength = mInfoDelta.mTotalBytes != -1;
+        final String transferEncoding = conn.getHeaderField("Transfer-Encoding");
+        final boolean isChunked = "chunked".equalsIgnoreCase(transferEncoding);
+        if (!hasLength && !isChunked) {
+            throw new StopRequestException(
+                    STATUS_CANNOT_RESUME, "can't know size of download, giving up");
+        }
+
         DrmManagerClient drmClient = null;
+        ParcelFileDescriptor outPfd = null;
+        FileDescriptor outFd = null;
         InputStream in = null;
         OutputStream out = null;
-        FileDescriptor outFd = null;
         try {
             try {
                 in = conn.getInputStream();
@@ -393,23 +446,49 @@ public class DownloadThread implements Runnable {
             }
 
             try {
-                if (DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType)) {
+                outPfd = mContext.getContentResolver()
+                        .openFileDescriptor(mInfo.getAllDownloadsUri(), "rw");
+                outFd = outPfd.getFileDescriptor();
+
+                if (DownloadDrmHelper.isDrmConvertNeeded(mInfoDelta.mMimeType)) {
                     drmClient = new DrmManagerClient(mContext);
-                    final RandomAccessFile file = new RandomAccessFile(
-                            new File(state.mFilename), "rw");
-                    out = new DrmOutputStream(drmClient, file, state.mMimeType);
-                    outFd = file.getFD();
+                    out = new DrmOutputStream(drmClient, outPfd, mInfoDelta.mMimeType);
                 } else {
-                    out = new FileOutputStream(state.mFilename, true);
-                    outFd = ((FileOutputStream) out).getFD();
+                    out = new ParcelFileDescriptor.AutoCloseOutputStream(outPfd);
+                }
+
+                // Pre-flight disk space requirements, when known
+                if (mInfoDelta.mTotalBytes > 0) {
+                    final long curSize = Libcore.os.fstat(outFd).st_size;
+                    final long newBytes = mInfoDelta.mTotalBytes - curSize;
+
+                    StorageUtils.ensureAvailableSpace(mContext, outFd, newBytes);
+
+                    try {
+                        // We found enough space, so claim it for ourselves
+                        Libcore.os.posix_fallocate(outFd, 0, mInfoDelta.mTotalBytes);
+                    } catch (ErrnoException e) {
+                        if (e.errno == OsConstants.ENOTSUP) {
+                            Log.w(TAG, "fallocate() said ENOTSUP; falling back to ftruncate()");
+                            Libcore.os.ftruncate(outFd, mInfoDelta.mTotalBytes);
+                        } else {
+                            throw e;
+                        }
+                    }
                 }
+
+                // Move into place to begin writing
+                Libcore.os.lseek(outFd, mInfoDelta.mCurrentBytes, SEEK_SET);
+
+            } catch (ErrnoException e) {
+                throw new StopRequestException(STATUS_FILE_ERROR, e);
             } catch (IOException e) {
                 throw new StopRequestException(STATUS_FILE_ERROR, e);
             }
 
             // Start streaming data, periodically watch for pause/cancel
             // commands and checking disk space as needed.
-            transferData(state, in, out);
+            transferData(in, out, outFd);
 
             try {
                 if (out instanceof DrmOutputStream) {
@@ -437,83 +516,137 @@ public class DownloadThread implements Runnable {
     }
 
     /**
-     * Check if current connectivity is valid for this request.
-     */
-    private void checkConnectivity() throws StopRequestException {
-        // checking connectivity will apply current policy
-        mPolicyDirty = false;
-
-        final NetworkState networkUsable = mInfo.checkCanUseNetwork();
-        if (networkUsable != NetworkState.OK) {
-            int status = Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
-            if (networkUsable == NetworkState.UNUSABLE_DUE_TO_SIZE) {
-                status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
-                mInfo.notifyPauseDueToSize(true);
-            } else if (networkUsable == NetworkState.RECOMMENDED_UNUSABLE_DUE_TO_SIZE) {
-                status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
-                mInfo.notifyPauseDueToSize(false);
-            }
-            throw new StopRequestException(status, networkUsable.name());
-        }
-    }
-
-    /**
      * Transfer as much data as possible from the HTTP response to the
      * destination file.
      */
-    private void transferData(State state, InputStream in, OutputStream out)
+    private void transferData(InputStream in, OutputStream out, FileDescriptor outFd)
             throws StopRequestException {
-        final byte data[] = new byte[Constants.BUFFER_SIZE];
-        for (;;) {
-            int bytesRead = readFromResponse(state, data, in);
-            if (bytesRead == -1) { // success, end of stream already reached
-                handleEndOfStream(state);
-                return;
+        final byte buffer[] = new byte[Constants.BUFFER_SIZE];
+        while (true) {
+            checkPausedOrCanceled();
+
+            int len = -1;
+            try {
+                len = in.read(buffer);
+            } catch (IOException e) {
+                throw new StopRequestException(
+                        STATUS_HTTP_DATA_ERROR, "Failed reading response: " + e, e);
+            }
+
+            if (len == -1) {
+                break;
             }
 
-            state.mGotData = true;
-            writeDataToDestination(state, data, bytesRead, out);
-            state.mCurrentBytes += bytesRead;
-            reportProgress(state);
+            try {
+                // When streaming, ensure space before each write
+                if (mInfoDelta.mTotalBytes == -1) {
+                    final long curSize = Libcore.os.fstat(outFd).st_size;
+                    final long newBytes = (mInfoDelta.mCurrentBytes + len) - curSize;
 
-            if (Constants.LOGVV) {
-                Log.v(Constants.TAG, "downloaded " + state.mCurrentBytes + " for "
-                      + mInfo.mUri);
+                    StorageUtils.ensureAvailableSpace(mContext, outFd, newBytes);
+                }
+
+                out.write(buffer, 0, len);
+
+                mMadeProgress = true;
+                mInfoDelta.mCurrentBytes += len;
+
+                updateProgress(outFd);
+
+            } catch (ErrnoException e) {
+                throw new StopRequestException(STATUS_FILE_ERROR, e);
+            } catch (IOException e) {
+                throw new StopRequestException(STATUS_FILE_ERROR, e);
             }
+        }
 
-            checkPausedOrCanceled(state);
+        // Finished without error; verify length if known
+        if (mInfoDelta.mTotalBytes != -1 && mInfoDelta.mCurrentBytes != mInfoDelta.mTotalBytes) {
+            throw new StopRequestException(STATUS_HTTP_DATA_ERROR, "Content length mismatch");
         }
     }
 
     /**
-     * Called after a successful completion to take any necessary action on the downloaded file.
+     * Called just before the thread finishes, regardless of status, to take any
+     * necessary action on the downloaded file.
      */
-    private void finalizeDestinationFile(State state) {
-        if (state.mFilename != null) {
-            // make sure the file is readable
-            FileUtils.setPermissions(state.mFilename, 0644, -1, -1);
+    private void finalizeDestination() {
+        if (Downloads.Impl.isStatusError(mInfoDelta.mStatus)) {
+            // When error, free up any disk space
+            try {
+                final ParcelFileDescriptor target = mContext.getContentResolver()
+                        .openFileDescriptor(mInfo.getAllDownloadsUri(), "rw");
+                try {
+                    Libcore.os.ftruncate(target.getFileDescriptor(), 0);
+                } catch (ErrnoException ignored) {
+                } finally {
+                    IoUtils.closeQuietly(target);
+                }
+            } catch (FileNotFoundException ignored) {
+            }
+
+            // Delete if local file
+            if (mInfoDelta.mFileName != null) {
+                new File(mInfoDelta.mFileName).delete();
+            }
+
+        } else if (Downloads.Impl.isStatusSuccess(mInfoDelta.mStatus)) {
+            // When success, open access if local file
+            if (mInfoDelta.mFileName != null) {
+                try {
+                    // TODO: remove this once PackageInstaller works with content://
+                    Libcore.os.chmod(mInfoDelta.mFileName, 0644);
+                } catch (ErrnoException ignored) {
+                }
+
+                if (mInfo.mDestination != Downloads.Impl.DESTINATION_FILE_URI) {
+                    try {
+                        // Move into final resting place, if needed
+                        final File before = new File(mInfoDelta.mFileName);
+                        final File beforeDir = Helpers.getRunningDestinationDirectory(
+                                mContext, mInfo.mDestination);
+                        final File afterDir = Helpers.getSuccessDestinationDirectory(
+                                mContext, mInfo.mDestination);
+                        if (!beforeDir.equals(afterDir)
+                                && before.getParentFile().equals(beforeDir)) {
+                            final File after = new File(afterDir, before.getName());
+                            if (before.renameTo(after)) {
+                                mInfoDelta.mFileName = after.getAbsolutePath();
+                            }
+                        }
+                    } catch (IOException ignored) {
+                    }
+                }
+            }
         }
     }
 
     /**
-     * Called just before the thread finishes, regardless of status, to take any necessary action on
-     * the downloaded file.
+     * Check if current connectivity is valid for this request.
      */
-    private void cleanupDestination(State state, int finalStatus) {
-        if (state.mFilename != null && Downloads.Impl.isStatusError(finalStatus)) {
-            if (Constants.LOGVV) {
-                Log.d(TAG, "cleanupDestination() deleting " + state.mFilename);
+    private void checkConnectivity() throws StopRequestException {
+        // checking connectivity will apply current policy
+        mPolicyDirty = false;
+
+        final NetworkState networkUsable = mInfo.checkCanUseNetwork(mInfoDelta.mTotalBytes);
+        if (networkUsable != NetworkState.OK) {
+            int status = Downloads.Impl.STATUS_WAITING_FOR_NETWORK;
+            if (networkUsable == NetworkState.UNUSABLE_DUE_TO_SIZE) {
+                status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
+                mInfo.notifyPauseDueToSize(true);
+            } else if (networkUsable == NetworkState.RECOMMENDED_UNUSABLE_DUE_TO_SIZE) {
+                status = Downloads.Impl.STATUS_QUEUED_FOR_WIFI;
+                mInfo.notifyPauseDueToSize(false);
             }
-            new File(state.mFilename).delete();
-            state.mFilename = null;
+            throw new StopRequestException(status, networkUsable.name());
         }
     }
 
     /**
-     * Check if the download has been paused or canceled, stopping the request appropriately if it
-     * has been.
+     * Check if the download has been paused or canceled, stopping the request
+     * appropriately if it has been.
      */
-    private void checkPausedOrCanceled(State state) throws StopRequestException {
+    private void checkPausedOrCanceled() throws StopRequestException {
         synchronized (mInfo) {
             if (mInfo.mControl == Downloads.Impl.CONTROL_PAUSED) {
                 throw new StopRequestException(
@@ -533,340 +666,133 @@ public class DownloadThread implements Runnable {
     /**
      * Report download progress through the database if necessary.
      */
-    private void reportProgress(State state) {
+    private void updateProgress(FileDescriptor outFd) throws IOException {
         final long now = SystemClock.elapsedRealtime();
+        final long currentBytes = mInfoDelta.mCurrentBytes;
 
-        final long sampleDelta = now - state.mSpeedSampleStart;
+        final long sampleDelta = now - mSpeedSampleStart;
         if (sampleDelta > 500) {
-            final long sampleSpeed = ((state.mCurrentBytes - state.mSpeedSampleBytes) * 1000)
+            final long sampleSpeed = ((currentBytes - mSpeedSampleBytes) * 1000)
                     / sampleDelta;
 
-            if (state.mSpeed == 0) {
-                state.mSpeed = sampleSpeed;
+            if (mSpeed == 0) {
+                mSpeed = sampleSpeed;
             } else {
-                state.mSpeed = ((state.mSpeed * 3) + sampleSpeed) / 4;
+                mSpeed = ((mSpeed * 3) + sampleSpeed) / 4;
             }
 
             // Only notify once we have a full sample window
-            if (state.mSpeedSampleStart != 0) {
-                mNotifier.notifyDownloadSpeed(mInfo.mId, state.mSpeed);
+            if (mSpeedSampleStart != 0) {
+                mNotifier.notifyDownloadSpeed(mId, mSpeed);
             }
 
-            state.mSpeedSampleStart = now;
-            state.mSpeedSampleBytes = state.mCurrentBytes;
+            mSpeedSampleStart = now;
+            mSpeedSampleBytes = currentBytes;
         }
 
-        if (state.mCurrentBytes - state.mBytesNotified > Constants.MIN_PROGRESS_STEP &&
-            now - state.mTimeLastNotification > Constants.MIN_PROGRESS_TIME) {
-            ContentValues values = new ContentValues();
-            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
-            mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
-            state.mBytesNotified = state.mCurrentBytes;
-            state.mTimeLastNotification = now;
-        }
-    }
-
-    /**
-     * Write a data buffer to the destination file.
-     * @param data buffer containing the data to write
-     * @param bytesRead how many bytes to write from the buffer
-     */
-    private void writeDataToDestination(State state, byte[] data, int bytesRead, OutputStream out)
-            throws StopRequestException {
-        mStorageManager.verifySpaceBeforeWritingToFile(
-                mInfo.mDestination, state.mFilename, bytesRead);
+        final long bytesDelta = currentBytes - mLastUpdateBytes;
+        final long timeDelta = now - mLastUpdateTime;
+        if (bytesDelta > Constants.MIN_PROGRESS_STEP && timeDelta > Constants.MIN_PROGRESS_TIME) {
+            // fsync() to ensure that current progress has been flushed to disk,
+            // so we can always resume based on latest database information.
+            outFd.sync();
 
-        boolean forceVerified = false;
-        while (true) {
-            try {
-                out.write(data, 0, bytesRead);
-                return;
-            } catch (IOException ex) {
-                // TODO: better differentiate between DRM and disk failures
-                if (!forceVerified) {
-                    // couldn't write to file. are we out of space? check.
-                    mStorageManager.verifySpace(mInfo.mDestination, state.mFilename, bytesRead);
-                    forceVerified = true;
-                } else {
-                    throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
-                            "Failed to write data: " + ex);
-                }
-            }
-        }
-    }
+            mInfoDelta.writeToDatabase();
 
-    /**
-     * Called when we've reached the end of the HTTP response stream, to update the database and
-     * check for consistency.
-     */
-    private void handleEndOfStream(State state) throws StopRequestException {
-        ContentValues values = new ContentValues();
-        values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
-        if (state.mContentLength == -1) {
-            values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, state.mCurrentBytes);
-        }
-        mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
-
-        final boolean lengthMismatched = (state.mContentLength != -1)
-                && (state.mCurrentBytes != state.mContentLength);
-        if (lengthMismatched) {
-            if (cannotResume(state)) {
-                throw new StopRequestException(STATUS_CANNOT_RESUME,
-                        "mismatched content length; unable to resume");
-            } else {
-                throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
-                        "closed socket before end of file");
-            }
+            mLastUpdateBytes = currentBytes;
+            mLastUpdateTime = now;
         }
     }
 
-    private boolean cannotResume(State state) {
-        return (state.mCurrentBytes > 0 && !mInfo.mNoIntegrity && state.mHeaderETag == null)
-                || DownloadDrmHelper.isDrmConvertNeeded(state.mMimeType);
-    }
-
     /**
-     * Read some data from the HTTP response stream, handling I/O errors.
-     * @param data buffer to use to read data
-     * @param entityStream stream for reading the HTTP response entity
-     * @return the number of bytes actually read or -1 if the end of the stream has been reached
+     * Process response headers from first server response. This derives its
+     * filename, size, and ETag.
      */
-    private int readFromResponse(State state, byte[] data, InputStream entityStream)
-            throws StopRequestException {
-        try {
-            return entityStream.read(data);
-        } catch (IOException ex) {
-            // TODO: handle stream errors the same as other retries
-            if ("unexpected end of stream".equals(ex.getMessage())) {
-                return -1;
-            }
+    private void parseOkHeaders(HttpURLConnection conn) throws StopRequestException {
+        if (mInfoDelta.mFileName == null) {
+            final String contentDisposition = conn.getHeaderField("Content-Disposition");
+            final String contentLocation = conn.getHeaderField("Content-Location");
 
-            ContentValues values = new ContentValues();
-            values.put(Downloads.Impl.COLUMN_CURRENT_BYTES, state.mCurrentBytes);
-            mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
-            if (cannotResume(state)) {
-                throw new StopRequestException(STATUS_CANNOT_RESUME,
-                        "Failed reading response: " + ex + "; unable to resume", ex);
-            } else {
-                throw new StopRequestException(STATUS_HTTP_DATA_ERROR,
-                        "Failed reading response: " + ex, ex);
+            try {
+                mInfoDelta.mFileName = Helpers.generateSaveFile(mContext, mInfoDelta.mUri,
+                        mInfo.mHint, contentDisposition, contentLocation, mInfoDelta.mMimeType,
+                        mInfo.mDestination);
+            } catch (IOException e) {
+                throw new StopRequestException(
+                        Downloads.Impl.STATUS_FILE_ERROR, "Failed to generate filename: " + e);
             }
         }
-    }
-
-    /**
-     * Prepare target file based on given network response. Derives filename and
-     * target size as needed.
-     */
-    private void processResponseHeaders(State state, HttpURLConnection conn)
-            throws StopRequestException {
-        // TODO: fallocate the entire file if header gave us specific length
-
-        readResponseHeaders(state, conn);
-
-        state.mFilename = Helpers.generateSaveFile(
-                mContext,
-                mInfo.mUri,
-                mInfo.mHint,
-                state.mContentDisposition,
-                state.mContentLocation,
-                state.mMimeType,
-                mInfo.mDestination,
-                state.mContentLength,
-                mStorageManager);
-
-        updateDatabaseFromHeaders(state);
-        // check connectivity again now that we know the total size
-        checkConnectivity();
-    }
-
-    /**
-     * Update necessary database fields based on values of HTTP response headers that have been
-     * read.
-     */
-    private void updateDatabaseFromHeaders(State state) {
-        ContentValues values = new ContentValues();
-        values.put(Downloads.Impl._DATA, state.mFilename);
-        if (state.mHeaderETag != null) {
-            values.put(Constants.ETAG, state.mHeaderETag);
-        }
-        if (state.mMimeType != null) {
-            values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
-        }
-        values.put(Downloads.Impl.COLUMN_TOTAL_BYTES, mInfo.mTotalBytes);
-        mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
-    }
-
-    /**
-     * Read headers from the HTTP response and store them into local state.
-     */
-    private void readResponseHeaders(State state, HttpURLConnection conn)
-            throws StopRequestException {
-        state.mContentDisposition = conn.getHeaderField("Content-Disposition");
-        state.mContentLocation = conn.getHeaderField("Content-Location");
 
-        if (state.mMimeType == null) {
-            state.mMimeType = Intent.normalizeMimeType(conn.getContentType());
+        if (mInfoDelta.mMimeType == null) {
+            mInfoDelta.mMimeType = Intent.normalizeMimeType(conn.getContentType());
         }
 
-        state.mHeaderETag = conn.getHeaderField("ETag");
-
         final String transferEncoding = conn.getHeaderField("Transfer-Encoding");
         if (transferEncoding == null) {
-            state.mContentLength = getHeaderFieldLong(conn, "Content-Length", -1);
+            mInfoDelta.mTotalBytes = getHeaderFieldLong(conn, "Content-Length", -1);
         } else {
-            Log.i(TAG, "Ignoring Content-Length since Transfer-Encoding is also defined");
-            state.mContentLength = -1;
+            mInfoDelta.mTotalBytes = -1;
         }
 
-        state.mTotalBytes = state.mContentLength;
-        mInfo.mTotalBytes = state.mContentLength;
+        mInfoDelta.mETag = conn.getHeaderField("ETag");
 
-        final boolean noSizeInfo = state.mContentLength == -1
-                && (transferEncoding == null || !transferEncoding.equalsIgnoreCase("chunked"));
-        if (!mInfo.mNoIntegrity && noSizeInfo) {
-            throw new StopRequestException(STATUS_CANNOT_RESUME,
-                    "can't know size of download, giving up");
-        }
+        mInfoDelta.writeToDatabase();
+
+        // Check connectivity again now that we know the total size
+        checkConnectivity();
     }
 
-    private void parseRetryAfterHeaders(State state, HttpURLConnection conn) {
-        state.mRetryAfter = conn.getHeaderFieldInt("Retry-After", -1);
-        if (state.mRetryAfter < 0) {
-            state.mRetryAfter = 0;
+    private void parseUnavailableHeaders(HttpURLConnection conn) {
+        long retryAfter = conn.getHeaderFieldInt("Retry-After", -1);
+        if (retryAfter < 0) {
+            retryAfter = 0;
         } else {
-            if (state.mRetryAfter < Constants.MIN_RETRY_AFTER) {
-                state.mRetryAfter = Constants.MIN_RETRY_AFTER;
-            } else if (state.mRetryAfter > Constants.MAX_RETRY_AFTER) {
-                state.mRetryAfter = Constants.MAX_RETRY_AFTER;
+            if (retryAfter < Constants.MIN_RETRY_AFTER) {
+                retryAfter = Constants.MIN_RETRY_AFTER;
+            } else if (retryAfter > Constants.MAX_RETRY_AFTER) {
+                retryAfter = Constants.MAX_RETRY_AFTER;
             }
-            state.mRetryAfter += Helpers.sRandom.nextInt(Constants.MIN_RETRY_AFTER + 1);
-            state.mRetryAfter *= 1000;
+            retryAfter += Helpers.sRandom.nextInt(Constants.MIN_RETRY_AFTER + 1);
         }
-    }
 
-    /**
-     * Prepare the destination file to receive data.  If the file already exists, we'll set up
-     * appropriately for resumption.
-     */
-    private void setupDestinationFile(State state) throws StopRequestException {
-        if (!TextUtils.isEmpty(state.mFilename)) { // only true if we've already run a thread for this download
-            if (Constants.LOGV) {
-                Log.i(Constants.TAG, "have run thread before for id: " + mInfo.mId +
-                        ", and state.mFilename: " + state.mFilename);
-            }
-            if (!Helpers.isFilenameValid(state.mFilename,
-                    mStorageManager.getDownloadDataDirectory())) {
-                // this should never happen
-                throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
-                        "found invalid internal destination filename");
-            }
-            // We're resuming a download that got interrupted
-            File f = new File(state.mFilename);
-            if (f.exists()) {
-                if (Constants.LOGV) {
-                    Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
-                            ", and state.mFilename: " + state.mFilename);
-                }
-                long fileLength = f.length();
-                if (fileLength == 0) {
-                    // The download hadn't actually started, we can restart from scratch
-                    if (Constants.LOGVV) {
-                        Log.d(TAG, "setupDestinationFile() found fileLength=0, deleting "
-                                + state.mFilename);
-                    }
-                    f.delete();
-                    state.mFilename = null;
-                    if (Constants.LOGV) {
-                        Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
-                                ", BUT starting from scratch again: ");
-                    }
-                } else if (mInfo.mETag == null && !mInfo.mNoIntegrity) {
-                    // This should've been caught upon failure
-                    if (Constants.LOGVV) {
-                        Log.d(TAG, "setupDestinationFile() unable to resume download, deleting "
-                                + state.mFilename);
-                    }
-                    f.delete();
-                    throw new StopRequestException(Downloads.Impl.STATUS_CANNOT_RESUME,
-                            "Trying to resume a download that can't be resumed");
-                } else {
-                    // All right, we'll be able to resume this download
-                    if (Constants.LOGV) {
-                        Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
-                                ", and starting with file of length: " + fileLength);
-                    }
-                    state.mCurrentBytes = (int) fileLength;
-                    if (mInfo.mTotalBytes != -1) {
-                        state.mContentLength = mInfo.mTotalBytes;
-                    }
-                    state.mHeaderETag = mInfo.mETag;
-                    state.mContinuingDownload = true;
-                    if (Constants.LOGV) {
-                        Log.i(Constants.TAG, "resuming download for id: " + mInfo.mId +
-                                ", state.mCurrentBytes: " + state.mCurrentBytes +
-                                ", and setting mContinuingDownload to true: ");
-                    }
-                }
-            }
-        }
+        mInfoDelta.mRetryAfter = (int) (retryAfter * SECOND_IN_MILLIS);
     }
 
     /**
      * Add custom headers for this download to the HTTP request.
      */
-    private void addRequestHeaders(State state, HttpURLConnection conn) {
+    private void addRequestHeaders(HttpURLConnection conn, boolean resuming) {
         for (Pair<String, String> header : mInfo.getHeaders()) {
             conn.addRequestProperty(header.first, header.second);
         }
 
         // Only splice in user agent when not already defined
         if (conn.getRequestProperty("User-Agent") == null) {
-            conn.addRequestProperty("User-Agent", userAgent());
+            conn.addRequestProperty("User-Agent", mInfo.getUserAgent());
         }
 
         // Defeat transparent gzip compression, since it doesn't allow us to
         // easily resume partial downloads.
         conn.setRequestProperty("Accept-Encoding", "identity");
 
-        if (state.mContinuingDownload) {
-            if (state.mHeaderETag != null) {
-                conn.addRequestProperty("If-Match", state.mHeaderETag);
+        if (resuming) {
+            if (mInfoDelta.mETag != null) {
+                conn.addRequestProperty("If-Match", mInfoDelta.mETag);
             }
-            conn.addRequestProperty("Range", "bytes=" + state.mCurrentBytes + "-");
+            conn.addRequestProperty("Range", "bytes=" + mInfoDelta.mCurrentBytes + "-");
         }
     }
 
-    /**
-     * Stores information about the completed download, and notifies the initiating application.
-     */
-    private void notifyDownloadCompleted(
-            State state, int finalStatus, String errorMsg, int numFailed) {
-        notifyThroughDatabase(state, finalStatus, errorMsg, numFailed);
-        if (Downloads.Impl.isStatusCompleted(finalStatus)) {
-            mInfo.sendIntentIfRequested();
-        }
+    private void logDebug(String msg) {
+        Log.d(TAG, "[" + mId + "] " + msg);
     }
 
-    private void notifyThroughDatabase(
-            State state, int finalStatus, String errorMsg, int numFailed) {
-        ContentValues values = new ContentValues();
-        values.put(Downloads.Impl.COLUMN_STATUS, finalStatus);
-        values.put(Downloads.Impl._DATA, state.mFilename);
-        values.put(Downloads.Impl.COLUMN_MIME_TYPE, state.mMimeType);
-        values.put(Downloads.Impl.COLUMN_LAST_MODIFICATION, mSystemFacade.currentTimeMillis());
-        values.put(Downloads.Impl.COLUMN_FAILED_CONNECTIONS, numFailed);
-        values.put(Constants.RETRY_AFTER_X_REDIRECT_COUNT, state.mRetryAfter);
-
-        if (!TextUtils.equals(mInfo.mUri, state.mRequestUri)) {
-            values.put(Downloads.Impl.COLUMN_URI, state.mRequestUri);
-        }
+    private void logWarning(String msg) {
+        Log.w(TAG, "[" + mId + "] " + msg);
+    }
 
-        // save the error message. could be useful to developers.
-        if (!TextUtils.isEmpty(errorMsg)) {
-            values.put(Downloads.Impl.COLUMN_ERROR_MSG, errorMsg);
-        }
-        mContext.getContentResolver().update(mInfo.getAllDownloadsUri(), values, null, null);
+    private void logError(String msg, Throwable t) {
+        Log.e(TAG, "[" + mId + "] " + msg, t);
     }
 
     private INetworkPolicyListener mPolicyListener = new INetworkPolicyListener.Stub() {
@@ -891,7 +817,7 @@ public class DownloadThread implements Runnable {
         }
     };
 
-    public static long getHeaderFieldLong(URLConnection conn, String field, long defaultValue) {
+    private static long getHeaderFieldLong(URLConnection conn, String field, long defaultValue) {
         try {
             return Long.parseLong(conn.getHeaderField(field));
         } catch (NumberFormatException e) {
index 3562dac..eb07139 100644 (file)
@@ -21,6 +21,7 @@ import static com.android.providers.downloads.Constants.TAG;
 import android.content.Context;
 import android.net.Uri;
 import android.os.Environment;
+import android.os.FileUtils;
 import android.os.SystemClock;
 import android.provider.Downloads;
 import android.util.Log;
@@ -68,98 +69,79 @@ public class Helpers {
 
     /**
      * Creates a filename (where the file should be saved) from info about a download.
+     * This file will be touched to reserve it.
      */
-    static String generateSaveFile(
-            Context context,
-            String url,
-            String hint,
-            String contentDisposition,
-            String contentLocation,
-            String mimeType,
-            int destination,
-            long contentLength,
-            StorageManager storageManager) throws StopRequestException {
-        if (contentLength < 0) {
-            contentLength = 0;
-        }
-        String path;
-        File base = null;
+    static String generateSaveFile(Context context, String url, String hint,
+            String contentDisposition, String contentLocation, String mimeType, int destination)
+            throws IOException {
+
+        final File parent;
+        final File[] parentTest;
+        String name = null;
+
         if (destination == Downloads.Impl.DESTINATION_FILE_URI) {
-            path = Uri.parse(hint).getPath();
+            final File file = new File(Uri.parse(hint).getPath());
+            parent = file.getParentFile().getAbsoluteFile();
+            parentTest = new File[] { parent };
+            name = file.getName();
         } else {
-            base = storageManager.locateDestinationDirectory(mimeType, destination,
-                    contentLength);
-            path = chooseFilename(url, hint, contentDisposition, contentLocation,
-                                             destination);
+            parent = getRunningDestinationDirectory(context, destination);
+            parentTest = new File[] {
+                    parent,
+                    getSuccessDestinationDirectory(context, destination)
+            };
+            name = chooseFilename(url, hint, contentDisposition, contentLocation);
+        }
+
+        // Ensure target directories are ready
+        for (File test : parentTest) {
+            if (!(test.isDirectory() || test.mkdirs())) {
+                throw new IOException("Failed to create parent for " + test);
+            }
         }
-        storageManager.verifySpace(destination, path, contentLength);
+
         if (DownloadDrmHelper.isDrmConvertNeeded(mimeType)) {
-            path = DownloadDrmHelper.modifyDrmFwLockFileExtension(path);
+            name = DownloadDrmHelper.modifyDrmFwLockFileExtension(name);
         }
-        path = getFullPath(path, mimeType, destination, base);
-        return path;
-    }
 
-    static String getFullPath(String filename, String mimeType, int destination, File base)
-            throws StopRequestException {
-        String extension = null;
-        int dotIndex = filename.lastIndexOf('.');
-        boolean missingExtension = dotIndex < 0 || dotIndex < filename.lastIndexOf('/');
+        final String prefix;
+        final String suffix;
+        final int dotIndex = name.lastIndexOf('.');
+        final boolean missingExtension = dotIndex < 0;
         if (destination == Downloads.Impl.DESTINATION_FILE_URI) {
             // Destination is explicitly set - do not change the extension
             if (missingExtension) {
-                extension = "";
+                prefix = name;
+                suffix = "";
             } else {
-                extension = filename.substring(dotIndex);
-                filename = filename.substring(0, dotIndex);
+                prefix = name.substring(0, dotIndex);
+                suffix = name.substring(dotIndex);
             }
         } else {
             // Split filename between base and extension
             // Add an extension if filename does not have one
             if (missingExtension) {
-                extension = chooseExtensionFromMimeType(mimeType, true);
+                prefix = name;
+                suffix = chooseExtensionFromMimeType(mimeType, true);
             } else {
-                extension = chooseExtensionFromFilename(mimeType, destination, filename, dotIndex);
-                filename = filename.substring(0, dotIndex);
+                prefix = name.substring(0, dotIndex);
+                suffix = chooseExtensionFromFilename(mimeType, destination, name, dotIndex);
             }
         }
 
-        boolean recoveryDir = Constants.RECOVERY_DIRECTORY.equalsIgnoreCase(filename + extension);
-
-        if (base != null) {
-            filename = base.getPath() + File.separator + filename;
-        }
-
-        if (Constants.LOGVV) {
-            Log.v(Constants.TAG, "target file: " + filename + extension);
-        }
-
         synchronized (sUniqueLock) {
-            final String path = chooseUniqueFilenameLocked(
-                    destination, filename, extension, recoveryDir);
+            name = generateAvailableFilenameLocked(parentTest, prefix, suffix);
 
             // Claim this filename inside lock to prevent other threads from
             // clobbering us. We're not paranoid enough to use O_EXCL.
-            try {
-                File file = new File(path);
-                File parent = file.getParentFile();
-
-                // Make sure the parent directories exists before generates new file
-                if (parent != null && !parent.exists()) {
-                    parent.mkdirs();
-                }
-
-                file.createNewFile();
-            } catch (IOException e) {
-                throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
-                        "Failed to create target file " + path, e);
-            }
-            return path;
+            final File file = new File(parent, name);
+            file.createNewFile();
+            return file.getAbsolutePath();
         }
     }
 
     private static String chooseFilename(String url, String hint, String contentDisposition,
-            String contentLocation, int destination) {
+            String contentLocation) {
         String filename = null;
 
         // First, try to use the hint from the application, if there's one
@@ -305,18 +287,25 @@ public class Helpers {
         return extension;
     }
 
-    private static String chooseUniqueFilenameLocked(int destination, String filename,
-            String extension, boolean recoveryDir) throws StopRequestException {
-        String fullFilename = filename + extension;
-        if (!new File(fullFilename).exists()
-                && (!recoveryDir ||
-                (destination != Downloads.Impl.DESTINATION_CACHE_PARTITION &&
-                        destination != Downloads.Impl.DESTINATION_SYSTEMCACHE_PARTITION &&
-                        destination != Downloads.Impl.DESTINATION_CACHE_PARTITION_PURGEABLE &&
-                        destination != Downloads.Impl.DESTINATION_CACHE_PARTITION_NOROAMING))) {
-            return fullFilename;
-        }
-        filename = filename + Constants.FILENAME_SEQUENCE_SEPARATOR;
+    private static boolean isFilenameAvailableLocked(File[] parents, String name) {
+        if (Constants.RECOVERY_DIRECTORY.equalsIgnoreCase(name)) return false;
+
+        for (File parent : parents) {
+            if (new File(parent, name).exists()) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private static String generateAvailableFilenameLocked(
+            File[] parents, String prefix, String suffix) throws IOException {
+        String name = prefix + suffix;
+        if (isFilenameAvailableLocked(parents, name)) {
+            return name;
+        }
+
         /*
         * This number is used to generate partially randomized filenames to avoid
         * collisions.
@@ -334,39 +323,38 @@ public class Helpers {
         int sequence = 1;
         for (int magnitude = 1; magnitude < 1000000000; magnitude *= 10) {
             for (int iteration = 0; iteration < 9; ++iteration) {
-                fullFilename = filename + sequence + extension;
-                if (!new File(fullFilename).exists()) {
-                    return fullFilename;
-                }
-                if (Constants.LOGVV) {
-                    Log.v(Constants.TAG, "file with sequence number " + sequence + " exists");
+                name = prefix + Constants.FILENAME_SEQUENCE_SEPARATOR + sequence + suffix;
+                if (isFilenameAvailableLocked(parents, name)) {
+                    return name;
                 }
                 sequence += sRandom.nextInt(magnitude) + 1;
             }
         }
-        throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
-                "failed to generate an unused filename on internal download storage");
+
+        throw new IOException("Failed to generate an available filename");
     }
 
     /**
-     * Checks whether the filename looks legitimate
+     * Checks whether the filename looks legitimate for security purposes. This
+     * prevents us from opening files that aren't actually downloads.
      */
-    static boolean isFilenameValid(String filename, File downloadsDataDir) {
-        final String[] whitelist;
+    static boolean isFilenameValid(Context context, File file) {
+        final File[] whitelist;
         try {
-            filename = new File(filename).getCanonicalPath();
-            whitelist = new String[] {
-                    downloadsDataDir.getCanonicalPath(),
-                    Environment.getDownloadCacheDirectory().getCanonicalPath(),
-                    Environment.getExternalStorageDirectory().getCanonicalPath(),
+            file = file.getCanonicalFile();
+            whitelist = new File[] {
+                    context.getFilesDir().getCanonicalFile(),
+                    context.getCacheDir().getCanonicalFile(),
+                    Environment.getDownloadCacheDirectory().getCanonicalFile(),
+                    Environment.getExternalStorageDirectory().getCanonicalFile(),
             };
         } catch (IOException e) {
             Log.w(TAG, "Failed to resolve canonical path: " + e);
             return false;
         }
 
-        for (String test : whitelist) {
-            if (filename.startsWith(test)) {
+        for (File testDir : whitelist) {
+            if (FileUtils.contains(testDir, file)) {
                 return true;
             }
         }
@@ -374,6 +362,49 @@ public class Helpers {
         return false;
     }
 
+    public static File getRunningDestinationDirectory(Context context, int destination)
+            throws IOException {
+        return getDestinationDirectory(context, destination, true);
+    }
+
+    public static File getSuccessDestinationDirectory(Context context, int destination)
+            throws IOException {
+        return getDestinationDirectory(context, destination, false);
+    }
+
+    private static File getDestinationDirectory(Context context, int destination, boolean running)
+            throws IOException {
+        switch (destination) {
+            case Downloads.Impl.DESTINATION_CACHE_PARTITION:
+            case Downloads.Impl.DESTINATION_CACHE_PARTITION_PURGEABLE:
+            case Downloads.Impl.DESTINATION_CACHE_PARTITION_NOROAMING:
+                if (running) {
+                    return context.getFilesDir();
+                } else {
+                    return context.getCacheDir();
+                }
+
+            case Downloads.Impl.DESTINATION_SYSTEMCACHE_PARTITION:
+                if (running) {
+                    return new File(Environment.getDownloadCacheDirectory(),
+                            Constants.DIRECTORY_CACHE_RUNNING);
+                } else {
+                    return Environment.getDownloadCacheDirectory();
+                }
+
+            case Downloads.Impl.DESTINATION_EXTERNAL:
+                final File target = new File(
+                        Environment.getExternalStorageDirectory(), Environment.DIRECTORY_DOWNLOADS);
+                if (!target.isDirectory() && target.mkdirs()) {
+                    throw new IOException("unable to create external downloads directory");
+                }
+                return target;
+
+            default:
+                throw new IllegalStateException("unexpected destination: " + destination);
+        }
+    }
+
     /**
      * Checks whether this looks like a legitimate selection parameter
      */
index a2b642d..07bd628 100644 (file)
@@ -34,13 +34,13 @@ class StopRequestException extends Exception {
     }
 
     public StopRequestException(int finalStatus, Throwable t) {
-        super(t);
-        mFinalStatus = finalStatus;
+        this(finalStatus, t.getMessage());
+        initCause(t);
     }
 
     public StopRequestException(int finalStatus, String message, Throwable t) {
-        super(message, t);
-        mFinalStatus = finalStatus;
+        this(finalStatus, message);
+        initCause(t);
     }
 
     public int getFinalStatus() {
diff --git a/src/com/android/providers/downloads/StorageManager.java b/src/com/android/providers/downloads/StorageManager.java
deleted file mode 100644 (file)
index deb412e..0000000
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Copyright (C) 2010 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.android.providers.downloads;
-
-import static com.android.providers.downloads.Constants.LOGV;
-import static com.android.providers.downloads.Constants.TAG;
-
-import android.content.ContentUris;
-import android.content.Context;
-import android.content.res.Resources;
-import android.database.Cursor;
-import android.database.sqlite.SQLiteException;
-import android.net.Uri;
-import android.os.Environment;
-import android.os.StatFs;
-import android.provider.Downloads;
-import android.text.TextUtils;
-import android.util.Log;
-
-import com.android.internal.R;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import libcore.io.ErrnoException;
-import libcore.io.Libcore;
-import libcore.io.StructStat;
-
-/**
- * Manages the storage space consumed by Downloads Data dir. When space falls below
- * a threshold limit (set in resource xml files), starts cleanup of the Downloads data dir
- * to free up space.
- */
-class StorageManager {
-    /** the max amount of space allowed to be taken up by the downloads data dir */
-    private static final long sMaxdownloadDataDirSize =
-            Resources.getSystem().getInteger(R.integer.config_downloadDataDirSize) * 1024 * 1024;
-
-    /** threshold (in bytes) beyond which the low space warning kicks in and attempt is made to
-     * purge some downloaded files to make space
-     */
-    private static final long sDownloadDataDirLowSpaceThreshold =
-            Resources.getSystem().getInteger(
-                    R.integer.config_downloadDataDirLowSpaceThreshold)
-                    * sMaxdownloadDataDirSize / 100;
-
-    /** see {@link Environment#getExternalStorageDirectory()} */
-    private final File mExternalStorageDir;
-
-    /** see {@link Environment#getDownloadCacheDirectory()} */
-    private final File mSystemCacheDir;
-
-    /** The downloaded files are saved to this dir. it is the value returned by
-     * {@link Context#getCacheDir()}.
-     */
-    private final File mDownloadDataDir;
-
-    /** how often do we need to perform checks on space to make sure space is available */
-    private static final int FREQUENCY_OF_CHECKS_ON_SPACE_AVAILABILITY = 1024 * 1024; // 1MB
-    private int mBytesDownloadedSinceLastCheckOnSpace = 0;
-
-    /** misc members */
-    private final Context mContext;
-
-    public StorageManager(Context context) {
-        mContext = context;
-        mDownloadDataDir = getDownloadDataDirectory(context);
-        mExternalStorageDir = Environment.getExternalStorageDirectory();
-        mSystemCacheDir = Environment.getDownloadCacheDirectory();
-        startThreadToCleanupDatabaseAndPurgeFileSystem();
-    }
-
-    /** How often should database and filesystem be cleaned up to remove spurious files
-     * from the file system and
-     * The value is specified in terms of num of downloads since last time the cleanup was done.
-     */
-    private static final int FREQUENCY_OF_DATABASE_N_FILESYSTEM_CLEANUP = 250;
-    private int mNumDownloadsSoFar = 0;
-
-    synchronized void incrementNumDownloadsSoFar() {
-        if (++mNumDownloadsSoFar % FREQUENCY_OF_DATABASE_N_FILESYSTEM_CLEANUP == 0) {
-            startThreadToCleanupDatabaseAndPurgeFileSystem();
-        }
-    }
-    /* start a thread to cleanup the following
-     *      remove spurious files from the file system
-     *      remove excess entries from the database
-     */
-    private Thread mCleanupThread = null;
-    private synchronized void startThreadToCleanupDatabaseAndPurgeFileSystem() {
-        if (mCleanupThread != null && mCleanupThread.isAlive()) {
-            return;
-        }
-        mCleanupThread = new Thread() {
-            @Override public void run() {
-                removeSpuriousFiles();
-                trimDatabase();
-            }
-        };
-        mCleanupThread.start();
-    }
-
-    void verifySpaceBeforeWritingToFile(int destination, String path, long length)
-            throws StopRequestException {
-        // do this check only once for every 1MB of downloaded data
-        if (incrementBytesDownloadedSinceLastCheckOnSpace(length) <
-                FREQUENCY_OF_CHECKS_ON_SPACE_AVAILABILITY) {
-            return;
-        }
-        verifySpace(destination, path, length);
-    }
-
-    void verifySpace(int destination, String path, long length) throws StopRequestException {
-        resetBytesDownloadedSinceLastCheckOnSpace();
-        File dir = null;
-        if (Constants.LOGV) {
-            Log.i(Constants.TAG, "in verifySpace, destination: " + destination +
-                    ", path: " + path + ", length: " + length);
-        }
-        if (path == null) {
-            throw new IllegalArgumentException("path can't be null");
-        }
-        switch (destination) {
-            case Downloads.Impl.DESTINATION_CACHE_PARTITION:
-            case Downloads.Impl.DESTINATION_CACHE_PARTITION_NOROAMING:
-            case Downloads.Impl.DESTINATION_CACHE_PARTITION_PURGEABLE:
-                dir = mDownloadDataDir;
-                break;
-            case Downloads.Impl.DESTINATION_EXTERNAL:
-                dir = mExternalStorageDir;
-                break;
-            case Downloads.Impl.DESTINATION_SYSTEMCACHE_PARTITION:
-                dir = mSystemCacheDir;
-                break;
-            case Downloads.Impl.DESTINATION_FILE_URI:
-                if (path.startsWith(mExternalStorageDir.getPath())) {
-                    dir = mExternalStorageDir;
-                } else if (path.startsWith(mDownloadDataDir.getPath())) {
-                    dir = mDownloadDataDir;
-                } else if (path.startsWith(mSystemCacheDir.getPath())) {
-                    dir = mSystemCacheDir;
-                }
-                break;
-         }
-        if (dir == null) {
-            throw new IllegalStateException("invalid combination of destination: " + destination +
-                    ", path: " + path);
-        }
-        findSpace(dir, length, destination);
-    }
-
-    /**
-     * finds space in the given filesystem (input param: root) to accommodate # of bytes
-     * specified by the input param(targetBytes).
-     * returns true if found. false otherwise.
-     */
-    private synchronized void findSpace(File root, long targetBytes, int destination)
-            throws StopRequestException {
-        if (targetBytes == 0) {
-            return;
-        }
-        if (destination == Downloads.Impl.DESTINATION_FILE_URI ||
-                destination == Downloads.Impl.DESTINATION_EXTERNAL) {
-            if (!Environment.getExternalStorageState().equals(Environment.MEDIA_MOUNTED)) {
-                throw new StopRequestException(Downloads.Impl.STATUS_DEVICE_NOT_FOUND_ERROR,
-                        "external media not mounted");
-            }
-        }
-        // is there enough space in the file system of the given param 'root'.
-        long bytesAvailable = getAvailableBytesInFileSystemAtGivenRoot(root);
-        if (bytesAvailable < sDownloadDataDirLowSpaceThreshold) {
-            /* filesystem's available space is below threshold for low space warning.
-             * threshold typically is 10% of download data dir space quota.
-             * try to cleanup and see if the low space situation goes away.
-             */
-            discardPurgeableFiles(destination, sDownloadDataDirLowSpaceThreshold);
-            removeSpuriousFiles();
-            bytesAvailable = getAvailableBytesInFileSystemAtGivenRoot(root);
-            if (bytesAvailable < sDownloadDataDirLowSpaceThreshold) {
-                /*
-                 * available space is still below the threshold limit.
-                 *
-                 * If this is system cache dir, print a warning.
-                 * otherwise, don't allow downloading until more space
-                 * is available because downloadmanager shouldn't end up taking those last
-                 * few MB of space left on the filesystem.
-                 */
-                if (root.equals(mSystemCacheDir)) {
-                    Log.w(Constants.TAG, "System cache dir ('/cache') is running low on space." +
-                            "space available (in bytes): " + bytesAvailable);
-                } else {
-                    throw new StopRequestException(Downloads.Impl.STATUS_INSUFFICIENT_SPACE_ERROR,
-                            "space in the filesystem rooted at: " + root +
-                            " is below 10% availability. stopping this download.");
-                }
-            }
-        }
-        if (root.equals(mDownloadDataDir)) {
-            // this download is going into downloads data dir. check space in that specific dir.
-            bytesAvailable = getAvailableBytesInDownloadsDataDir(mDownloadDataDir);
-            if (bytesAvailable < sDownloadDataDirLowSpaceThreshold) {
-                // print a warning
-                Log.w(Constants.TAG, "Downloads data dir: " + root +
-                        " is running low on space. space available (in bytes): " + bytesAvailable);
-            }
-            if (bytesAvailable < targetBytes) {
-                // Insufficient space; make space.
-                discardPurgeableFiles(destination, sDownloadDataDirLowSpaceThreshold);
-                removeSpuriousFiles();
-                bytesAvailable = getAvailableBytesInDownloadsDataDir(mDownloadDataDir);
-            }
-        }
-        if (bytesAvailable < targetBytes) {
-            throw new StopRequestException(Downloads.Impl.STATUS_INSUFFICIENT_SPACE_ERROR,
-                    "not enough free space in the filesystem rooted at: " + root +
-                    " and unable to free any more");
-        }
-    }
-
-    /**
-     * returns the number of bytes available in the downloads data dir
-     * TODO this implementation is too slow. optimize it.
-     */
-    private long getAvailableBytesInDownloadsDataDir(File root) {
-        File[] files = root.listFiles();
-        long space = sMaxdownloadDataDirSize;
-        if (files == null) {
-            return space;
-        }
-        int size = files.length;
-        for (int i = 0; i < size; i++) {
-            space -= files[i].length();
-        }
-        if (Constants.LOGV) {
-            Log.i(Constants.TAG, "available space (in bytes) in downloads data dir: " + space);
-        }
-        return space;
-    }
-
-    private long getAvailableBytesInFileSystemAtGivenRoot(File root) {
-        StatFs stat = new StatFs(root.getPath());
-        // put a bit of margin (in case creating the file grows the system by a few blocks)
-        long availableBlocks = (long) stat.getAvailableBlocks() - 4;
-        long size = stat.getBlockSize() * availableBlocks;
-        if (Constants.LOGV) {
-            Log.i(Constants.TAG, "available space (in bytes) in filesystem rooted at: " +
-                    root.getPath() + " is: " + size);
-        }
-        return size;
-    }
-
-    File locateDestinationDirectory(String mimeType, int destination, long contentLength)
-            throws StopRequestException {
-        switch (destination) {
-            case Downloads.Impl.DESTINATION_CACHE_PARTITION:
-            case Downloads.Impl.DESTINATION_CACHE_PARTITION_PURGEABLE:
-            case Downloads.Impl.DESTINATION_CACHE_PARTITION_NOROAMING:
-                return mDownloadDataDir;
-            case Downloads.Impl.DESTINATION_SYSTEMCACHE_PARTITION:
-                return mSystemCacheDir;
-            case Downloads.Impl.DESTINATION_EXTERNAL:
-                File base = new File(mExternalStorageDir.getPath() + Constants.DEFAULT_DL_SUBDIR);
-                if (!base.isDirectory() && !base.mkdir()) {
-                    // Can't create download directory, e.g. because a file called "download"
-                    // already exists at the root level, or the SD card filesystem is read-only.
-                    throw new StopRequestException(Downloads.Impl.STATUS_FILE_ERROR,
-                            "unable to create external downloads directory " + base.getPath());
-                }
-                return base;
-            default:
-                throw new IllegalStateException("unexpected value for destination: " + destination);
-        }
-    }
-
-    File getDownloadDataDirectory() {
-        return mDownloadDataDir;
-    }
-
-    public static File getDownloadDataDirectory(Context context) {
-        return context.getCacheDir();
-    }
-
-    /**
-     * Deletes purgeable files from the cache partition. This also deletes
-     * the matching database entries. Files are deleted in LRU order until
-     * the total byte size is greater than targetBytes
-     */
-    private long discardPurgeableFiles(int destination, long targetBytes) {
-        if (true || Constants.LOGV) {
-            Log.i(Constants.TAG, "discardPurgeableFiles: destination = " + destination +
-                    ", targetBytes = " + targetBytes);
-        }
-        String destStr  = (destination == Downloads.Impl.DESTINATION_SYSTEMCACHE_PARTITION) ?
-                String.valueOf(destination) :
-                String.valueOf(Downloads.Impl.DESTINATION_CACHE_PARTITION_PURGEABLE);
-        String[] bindArgs = new String[]{destStr};
-        Cursor cursor = mContext.getContentResolver().query(
-                Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI,
-                null,
-                "( " +
-                Downloads.Impl.COLUMN_STATUS + " = '" + Downloads.Impl.STATUS_SUCCESS + "' AND " +
-                Downloads.Impl.COLUMN_DESTINATION + " = ? )",
-                bindArgs,
-                Downloads.Impl.COLUMN_LAST_MODIFICATION);
-        if (cursor == null) {
-            return 0;
-        }
-        long totalFreed = 0;
-        try {
-            final int dataIndex = cursor.getColumnIndex(Downloads.Impl._DATA);
-            while (cursor.moveToNext() && totalFreed < targetBytes) {
-                final String data = cursor.getString(dataIndex);
-                if (TextUtils.isEmpty(data)) continue;
-
-                File file = new File(data);
-                if (Constants.LOGV) {
-                    Log.d(Constants.TAG, "purging " + file.getAbsolutePath() + " for "
-                            + file.length() + " bytes");
-                }
-                totalFreed += file.length();
-                file.delete();
-                long id = cursor.getLong(cursor.getColumnIndex(Downloads.Impl._ID));
-                mContext.getContentResolver().delete(
-                        ContentUris.withAppendedId(Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI, id),
-                        null, null);
-            }
-        } finally {
-            cursor.close();
-        }
-        if (true || Constants.LOGV) {
-            Log.i(Constants.TAG, "Purged files, freed " + totalFreed + " for " +
-                    targetBytes + " requested");
-        }
-        return totalFreed;
-    }
-
-    /**
-     * Removes files in the systemcache and downloads data dir without corresponding entries in
-     * the downloads database.
-     * This can occur if a delete is done on the database but the file is not removed from the
-     * filesystem (due to sudden death of the process, for example).
-     * This is not a very common occurrence. So, do this only once in a while.
-     */
-    private void removeSpuriousFiles() {
-        if (Constants.LOGV) {
-            Log.i(Constants.TAG, "in removeSpuriousFiles");
-        }
-        // get a list of all files in system cache dir and downloads data dir
-        List<File> files = new ArrayList<File>();
-        File[] listOfFiles = mSystemCacheDir.listFiles();
-        if (listOfFiles != null) {
-            files.addAll(Arrays.asList(listOfFiles));
-        }
-        listOfFiles = mDownloadDataDir.listFiles();
-        if (listOfFiles != null) {
-            files.addAll(Arrays.asList(listOfFiles));
-        }
-        if (files.size() == 0) {
-            return;
-        }
-        Cursor cursor = mContext.getContentResolver().query(
-                Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI,
-                new String[] { Downloads.Impl._DATA }, null, null, null);
-        try {
-            if (cursor != null) {
-                while (cursor.moveToNext()) {
-                    String filename = cursor.getString(0);
-                    if (!TextUtils.isEmpty(filename)) {
-                        if (LOGV) {
-                            Log.i(Constants.TAG, "in removeSpuriousFiles, preserving file " +
-                                    filename);
-                        }
-                        files.remove(new File(filename));
-                    }
-                }
-            }
-        } finally {
-            if (cursor != null) {
-                cursor.close();
-            }
-        }
-
-        // delete files owned by us, but that don't appear in our database
-        final int myUid = android.os.Process.myUid();
-        for (File file : files) {
-            final String path = file.getAbsolutePath();
-            try {
-                final StructStat stat = Libcore.os.stat(path);
-                if (stat.st_uid == myUid) {
-                    if (Constants.LOGVV) {
-                        Log.d(TAG, "deleting spurious file " + path);
-                    }
-                    file.delete();
-                }
-            } catch (ErrnoException e) {
-                Log.w(TAG, "stat(" + path + ") result: " + e);
-            }
-        }
-    }
-
-    /**
-     * Drops old rows from the database to prevent it from growing too large
-     * TODO logic in this method needs to be optimized. maintain the number of downloads
-     * in memory - so that this method can limit the amount of data read.
-     */
-    private void trimDatabase() {
-        if (Constants.LOGV) {
-            Log.i(Constants.TAG, "in trimDatabase");
-        }
-        Cursor cursor = null;
-        try {
-            cursor = mContext.getContentResolver().query(Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI,
-                    new String[] { Downloads.Impl._ID },
-                    Downloads.Impl.COLUMN_STATUS + " >= '200'", null,
-                    Downloads.Impl.COLUMN_LAST_MODIFICATION);
-            if (cursor == null) {
-                // This isn't good - if we can't do basic queries in our database,
-                // nothing's gonna work
-                Log.e(Constants.TAG, "null cursor in trimDatabase");
-                return;
-            }
-            if (cursor.moveToFirst()) {
-                int numDelete = cursor.getCount() - Constants.MAX_DOWNLOADS;
-                int columnId = cursor.getColumnIndexOrThrow(Downloads.Impl._ID);
-                while (numDelete > 0) {
-                    Uri downloadUri = ContentUris.withAppendedId(
-                            Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI, cursor.getLong(columnId));
-                    mContext.getContentResolver().delete(downloadUri, null, null);
-                    if (!cursor.moveToNext()) {
-                        break;
-                    }
-                    numDelete--;
-                }
-            }
-        } catch (SQLiteException e) {
-            // trimming the database raised an exception. alright, ignore the exception
-            // and return silently. trimming database is not exactly a critical operation
-            // and there is no need to propagate the exception.
-            Log.w(Constants.TAG, "trimDatabase failed with exception: " + e.getMessage());
-            return;
-        } finally {
-            if (cursor != null) {
-                cursor.close();
-            }
-        }
-    }
-
-    private synchronized int incrementBytesDownloadedSinceLastCheckOnSpace(long val) {
-        mBytesDownloadedSinceLastCheckOnSpace += val;
-        return mBytesDownloadedSinceLastCheckOnSpace;
-    }
-
-    private synchronized void resetBytesDownloadedSinceLastCheckOnSpace() {
-        mBytesDownloadedSinceLastCheckOnSpace = 0;
-    }
-}
diff --git a/src/com/android/providers/downloads/StorageUtils.java b/src/com/android/providers/downloads/StorageUtils.java
new file mode 100644 (file)
index 0000000..53da8e1
--- /dev/null
@@ -0,0 +1,336 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.providers.downloads;
+
+import static android.net.TrafficStats.MB_IN_BYTES;
+import static android.provider.Downloads.Impl.STATUS_INSUFFICIENT_SPACE_ERROR;
+import static android.text.format.DateUtils.DAY_IN_MILLIS;
+import static com.android.providers.downloads.Constants.TAG;
+
+import android.app.DownloadManager;
+import android.content.ContentResolver;
+import android.content.ContentUris;
+import android.content.Context;
+import android.content.pm.IPackageDataObserver;
+import android.content.pm.PackageManager;
+import android.database.Cursor;
+import android.os.Environment;
+import android.provider.Downloads;
+import android.text.TextUtils;
+import android.util.Slog;
+
+import com.android.internal.annotations.VisibleForTesting;
+import com.google.android.collect.Lists;
+import com.google.android.collect.Sets;
+
+import libcore.io.ErrnoException;
+import libcore.io.IoUtils;
+import libcore.io.Libcore;
+import libcore.io.StructStat;
+import libcore.io.StructStatVfs;
+
+import java.io.File;
+import java.io.FileDescriptor;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utility methods for managing storage space related to
+ * {@link DownloadManager}.
+ */
+public class StorageUtils {
+
+    // TODO: run idle maint service to clean up untracked downloads
+
+    /**
+     * Minimum age for a file to be considered for deletion.
+     */
+    static final long MIN_DELETE_AGE = DAY_IN_MILLIS;
+
+    /**
+     * Reserved disk space to avoid filling disk.
+     */
+    static final long RESERVED_BYTES = 32 * MB_IN_BYTES;
+
+    @VisibleForTesting
+    static boolean sForceFullEviction = false;
+
+    /**
+     * Ensure that requested free space exists on the partition backing the
+     * given {@link FileDescriptor}. If not enough space is available, it tries
+     * freeing up space as follows:
+     * <ul>
+     * <li>If backed by the data partition (including emulated external
+     * storage), then ask {@link PackageManager} to free space from cache
+     * directories.
+     * <li>If backed by the cache partition, then try deleting older downloads
+     * to free space.
+     * </ul>
+     */
+    public static void ensureAvailableSpace(Context context, FileDescriptor fd, long bytes)
+            throws IOException, StopRequestException {
+
+        long availBytes = getAvailableBytes(fd);
+        if (availBytes >= bytes) {
+            // Underlying partition has enough space; go ahead
+            return;
+        }
+
+        // Not enough space, let's try freeing some up. Start by tracking down
+        // the backing partition.
+        final long dev;
+        try {
+            dev = Libcore.os.fstat(fd).st_dev;
+        } catch (ErrnoException e) {
+            throw e.rethrowAsIOException();
+        }
+
+        final long dataDev = getDeviceId(Environment.getDataDirectory());
+        final long cacheDev = getDeviceId(Environment.getDownloadCacheDirectory());
+        final long externalDev = getDeviceId(Environment.getExternalStorageDirectory());
+
+        if (dev == dataDev || (dev == externalDev && Environment.isExternalStorageEmulated())) {
+            // File lives on internal storage; ask PackageManager to try freeing
+            // up space from cache directories.
+            final PackageManager pm = context.getPackageManager();
+            final ObserverLatch observer = new ObserverLatch();
+            pm.freeStorageAndNotify(sForceFullEviction ? Long.MAX_VALUE : bytes, observer);
+
+            try {
+                if (!observer.latch.await(30, TimeUnit.SECONDS)) {
+                    throw new IOException("Timeout while freeing disk space");
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+
+        } else if (dev == cacheDev) {
+            // Try removing old files on cache partition
+            freeCacheStorage(bytes);
+        }
+
+        // Did we free enough space?
+        availBytes = getAvailableBytes(fd);
+        if (availBytes < bytes) {
+            throw new StopRequestException(STATUS_INSUFFICIENT_SPACE_ERROR,
+                    "Not enough free space; " + bytes + " requested, " + availBytes + " available");
+        }
+    }
+
+    /**
+     * Free requested space on cache partition, deleting oldest files first.
+     * We're only focused on freeing up disk space, and rely on the next orphan
+     * pass to clean up database entries.
+     */
+    private static void freeCacheStorage(long bytes) {
+        // Only consider finished downloads
+        final List<ConcreteFile> files = listFilesRecursive(
+                Environment.getDownloadCacheDirectory(), Constants.DIRECTORY_CACHE_RUNNING,
+                android.os.Process.myUid());
+
+        Slog.d(TAG, "Found " + files.size() + " downloads on cache");
+
+        Collections.sort(files, new Comparator<ConcreteFile>() {
+            @Override
+            public int compare(ConcreteFile lhs, ConcreteFile rhs) {
+                return (int) (lhs.file.lastModified() - rhs.file.lastModified());
+            }
+        });
+
+        final long now = System.currentTimeMillis();
+        for (ConcreteFile file : files) {
+            if (bytes <= 0) break;
+
+            if (now - file.file.lastModified() < MIN_DELETE_AGE) {
+                Slog.d(TAG, "Skipping recently modified " + file.file);
+            } else {
+                final long len = file.file.length();
+                Slog.d(TAG, "Deleting " + file.file + " to reclaim " + len);
+                bytes -= len;
+                file.file.delete();
+            }
+        }
+    }
+
+    private interface DownloadQuery {
+        final String[] PROJECTION = new String[] {
+                Downloads.Impl._ID,
+                Downloads.Impl._DATA };
+
+        final int _ID = 0;
+        final int _DATA = 1;
+    }
+
+    /**
+     * Clean up orphan downloads, both in database and on disk.
+     */
+    public static void cleanOrphans(Context context) {
+        final ContentResolver resolver = context.getContentResolver();
+
+        // Collect known files from database
+        final HashSet<ConcreteFile> fromDb = Sets.newHashSet();
+        final Cursor cursor = resolver.query(Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI,
+                DownloadQuery.PROJECTION, null, null, null);
+        try {
+            while (cursor.moveToNext()) {
+                final String path = cursor.getString(DownloadQuery._DATA);
+                if (TextUtils.isEmpty(path)) continue;
+
+                final File file = new File(path);
+                try {
+                    fromDb.add(new ConcreteFile(file));
+                } catch (ErrnoException e) {
+                    // File probably no longer exists
+                    final String state = Environment.getExternalStorageState(file);
+                    if (Environment.MEDIA_UNKNOWN.equals(state)
+                            || Environment.MEDIA_MOUNTED.equals(state)) {
+                        // File appears to live on internal storage, or a
+                        // currently mounted device, so remove it from database.
+                        // This preserves files on external storage while media
+                        // is removed.
+                        final long id = cursor.getLong(DownloadQuery._ID);
+                        Slog.d(TAG, "Missing " + file + ", deleting " + id);
+                        resolver.delete(ContentUris.withAppendedId(
+                                Downloads.Impl.ALL_DOWNLOADS_CONTENT_URI, id), null, null);
+                    }
+                }
+            }
+        } finally {
+            IoUtils.closeQuietly(cursor);
+        }
+
+        // Collect known files from disk
+        final int uid = android.os.Process.myUid();
+        final ArrayList<ConcreteFile> fromDisk = Lists.newArrayList();
+        fromDisk.addAll(listFilesRecursive(context.getCacheDir(), null, uid));
+        fromDisk.addAll(listFilesRecursive(context.getFilesDir(), null, uid));
+        fromDisk.addAll(listFilesRecursive(Environment.getDownloadCacheDirectory(), null, uid));
+
+        // Delete files no longer referenced by database
+        for (ConcreteFile file : fromDisk) {
+            if (!fromDb.contains(file)) {
+                Slog.d(TAG, "Missing db entry, deleting " + file.file);
+                file.file.delete();
+            }
+        }
+    }
+
+    /**
+     * Return number of available bytes on the filesystem backing the given
+     * {@link FileDescriptor}, minus any {@link #RESERVED_BYTES} buffer.
+     */
+    private static long getAvailableBytes(FileDescriptor fd) throws IOException {
+        try {
+            final StructStatVfs stat = Libcore.os.fstatvfs(fd);
+            return (stat.f_bavail * stat.f_bsize) - RESERVED_BYTES;
+        } catch (ErrnoException e) {
+            throw e.rethrowAsIOException();
+        }
+    }
+
+    private static long getDeviceId(File file) {
+        try {
+            return Libcore.os.stat(file.getAbsolutePath()).st_dev;
+        } catch (ErrnoException e) {
+            // Safe since dev_t is uint
+            return -1;
+        }
+    }
+
+    /**
+     * Return list of all normal files under the given directory, traversing
+     * directories recursively.
+     *
+     * @param exclude ignore dirs with this name, or {@code null} to ignore.
+     * @param uid only return files owned by this UID, or {@code -1} to ignore.
+     */
+    private static List<ConcreteFile> listFilesRecursive(File startDir, String exclude, int uid) {
+        final ArrayList<ConcreteFile> files = Lists.newArrayList();
+        final LinkedList<File> dirs = new LinkedList<File>();
+        dirs.add(startDir);
+        while (!dirs.isEmpty()) {
+            final File dir = dirs.removeFirst();
+            if (Objects.equals(dir.getName(), exclude)) continue;
+
+            final File[] children = dir.listFiles();
+            if (children == null) continue;
+
+            for (File child : children) {
+                if (child.isDirectory()) {
+                    dirs.add(child);
+                } else if (child.isFile()) {
+                    try {
+                        final ConcreteFile file = new ConcreteFile(child);
+                        if (uid == -1 || file.stat.st_uid == uid) {
+                            files.add(file);
+                        }
+                    } catch (ErrnoException ignored) {
+                    }
+                }
+            }
+        }
+        return files;
+    }
+
+    /**
+     * Concrete file on disk that has a backing device and inode. Faster than
+     * {@code realpath()} when looking for identical files.
+     */
+    public static class ConcreteFile {
+        public final File file;
+        public final StructStat stat;
+
+        public ConcreteFile(File file) throws ErrnoException {
+            this.file = file;
+            this.stat = Libcore.os.lstat(file.getAbsolutePath());
+        }
+
+        @Override
+        public int hashCode() {
+            int result = 1;
+            result = 31 * result + (int) (stat.st_dev ^ (stat.st_dev >>> 32));
+            result = 31 * result + (int) (stat.st_ino ^ (stat.st_ino >>> 32));
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (o instanceof ConcreteFile) {
+                final ConcreteFile f = (ConcreteFile) o;
+                return (f.stat.st_dev == stat.st_dev) && (f.stat.st_ino == stat.st_ino);
+            }
+            return false;
+        }
+    }
+
+    static class ObserverLatch extends IPackageDataObserver.Stub {
+        public final CountDownLatch latch = new CountDownLatch(1);
+
+        @Override
+        public void onRemoveCompleted(String packageName, boolean succeeded) {
+            latch.countDown();
+        }
+    }
+}
index d520123..ec73ca2 100644 (file)
@@ -19,6 +19,8 @@
           package="com.android.providers.downloads.tests"
           android:sharedUserId="android.media">
 
+    <uses-permission android:name="android.permission.ACCESS_DOWNLOAD_MANAGER_ADVANCED" />
+
     <application>
         <uses-library android:name="android.test.runner" />
     </application>
index 3b93738..28c5dc7 100644 (file)
@@ -74,15 +74,18 @@ public abstract class AbstractDownloadProviderFunctionalTest extends
     static class MockContentResolverWithNotify extends MockContentResolver {
         public boolean mNotifyWasCalled = false;
 
+        public MockContentResolverWithNotify(Context context) {
+            super(context);
+        }
+
         public synchronized void resetNotified() {
             mNotifyWasCalled = false;
         }
 
         @Override
-        public synchronized void notifyChange(Uri uri, ContentObserver observer,
-                boolean syncToNetwork) {
+        public synchronized void notifyChange(
+                Uri uri, ContentObserver observer, boolean syncToNetwork) {
             mNotifyWasCalled = true;
-            notifyAll();
         }
     }
 
@@ -94,20 +97,17 @@ public abstract class AbstractDownloadProviderFunctionalTest extends
     static class TestContext extends RenamingDelegatingContext {
         private static final String FILENAME_PREFIX = "test.";
 
-        private ContentResolver mResolver;
+        private final ContentResolver mResolver;
         private final NotificationManager mNotifManager;
 
         boolean mHasServiceBeenStarted = false;
 
         public TestContext(Context realContext) {
             super(realContext, FILENAME_PREFIX);
+            mResolver = new MockContentResolverWithNotify(this);
             mNotifManager = mock(NotificationManager.class);
         }
 
-        public void setResolver(ContentResolver resolver) {
-            mResolver = resolver;
-        }
-
         /**
          * Direct DownloadService to our test instance of DownloadProvider.
          */
@@ -156,12 +156,20 @@ public abstract class AbstractDownloadProviderFunctionalTest extends
         System.setProperty("dexmaker.dexcache", getContext().getCacheDir().toString());
 
         final Context realContext = getContext();
+
         mTestContext = new TestContext(realContext);
-        setupProviderAndResolver();
-        mTestContext.setResolver(mResolver);
+        mResolver = (MockContentResolverWithNotify) mTestContext.getContentResolver();
+
+        final DownloadProvider provider = new DownloadProvider();
+        provider.mSystemFacade = mSystemFacade;
+        provider.attachInfo(mTestContext, null);
+
+        mResolver.addProvider(PROVIDER_AUTHORITY, provider);
+
         setContext(mTestContext);
         setupService();
         getService().mSystemFacade = mSystemFacade;
+
         mSystemFacade.setUp();
         assertTrue(isDatabaseEmpty()); // ensure we're not messing with real data
         mServer = new MockWebServer();
@@ -186,14 +194,6 @@ public abstract class AbstractDownloadProviderFunctionalTest extends
         }
     }
 
-    void setupProviderAndResolver() {
-        DownloadProvider provider = new DownloadProvider();
-        provider.mSystemFacade = mSystemFacade;
-        provider.attachInfo(mTestContext, null);
-        mResolver = new MockContentResolverWithNotify();
-        mResolver.addProvider(PROVIDER_AUTHORITY, provider);
-    }
-
     /**
      * Remove any downloaded files and delete any lingering downloads.
      */
index 348dbd1..2846c7a 100644 (file)
@@ -28,6 +28,9 @@ import android.os.ParcelFileDescriptor;
 import android.os.SystemClock;
 import android.util.Log;
 
+import libcore.io.IoUtils;
+import libcore.io.Streams;
+
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.UnknownHostException;
@@ -91,19 +94,23 @@ public abstract class AbstractPublicApiTest extends AbstractDownloadProviderFunc
             }
         }
 
-        String getContents() throws Exception {
+        byte[] getRawContents() throws Exception {
             ParcelFileDescriptor downloadedFile = mManager.openDownloadedFile(mId);
             assertTrue("Invalid file descriptor: " + downloadedFile,
                        downloadedFile.getFileDescriptor().valid());
-            final InputStream stream = new ParcelFileDescriptor.AutoCloseInputStream(
+            final InputStream is = new ParcelFileDescriptor.AutoCloseInputStream(
                     downloadedFile);
             try {
-                return readStream(stream);
+                return Streams.readFully(is);
             } finally {
-                stream.close();
+                IoUtils.closeQuietly(is);
             }
         }
 
+        String getContents() throws Exception {
+            return new String(getRawContents());
+        }
+
         void runUntilStatus(int status) throws TimeoutException {
             final long startMillis = mSystemFacade.currentTimeMillis();
             startService(null);
index d54c122..5a15d39 100644 (file)
@@ -64,12 +64,12 @@ public class FakeSystemFacade implements SystemFacade {
 
     @Override
     public Long getMaxBytesOverMobile() {
-        return mMaxBytesOverMobile ;
+        return mMaxBytesOverMobile;
     }
 
     @Override
     public Long getRecommendedMaxBytesOverMobile() {
-        return mRecommendedMaxBytesOverMobile ;
+        return mRecommendedMaxBytesOverMobile;
     }
 
     @Override
index 50f4c44..121b7cd 100644 (file)
 
 package com.android.providers.downloads;
 
+import android.net.Uri;
 import android.provider.Downloads;
 import android.test.AndroidTestCase;
-import android.test.suitebuilder.annotation.LargeTest;
+import android.test.suitebuilder.annotation.SmallTest;
+
+import libcore.io.IoUtils;
+
+import java.io.File;
 
 /**
  * This test exercises methods in the {@Helpers} utility class.
  */
-@LargeTest
+@SmallTest
 public class HelpersTest extends AndroidTestCase {
 
-    public HelpersTest() {
+    @Override
+    protected void tearDown() throws Exception {
+        IoUtils.deleteContents(getContext().getFilesDir());
+        IoUtils.deleteContents(getContext().getCacheDir());
+
+        super.tearDown();
+    }
+
+    public void testGenerateSaveFile() throws Exception {
+        final File expected = new File(getContext().getFilesDir(), "file.mp4");
+        final String actual = Helpers.generateSaveFile(getContext(),
+                "http://example.com/file.txt", null, null, null,
+                "video/mp4", Downloads.Impl.DESTINATION_CACHE_PARTITION);
+        assertEquals(expected.getAbsolutePath(), actual);
+    }
+
+    public void testGenerateSaveFileDupes() throws Exception {
+        final File expected1 = new File(getContext().getFilesDir(), "file.txt");
+        final String actual1 = Helpers.generateSaveFile(getContext(), "http://example.com/file.txt",
+                null, null, null, null, Downloads.Impl.DESTINATION_CACHE_PARTITION);
+
+        final File expected2 = new File(getContext().getFilesDir(), "file-1.txt");
+        final String actual2 = Helpers.generateSaveFile(getContext(), "http://example.com/file.txt",
+                null, null, null, null, Downloads.Impl.DESTINATION_CACHE_PARTITION);
+
+        assertEquals(expected1.getAbsolutePath(), actual1);
+        assertEquals(expected2.getAbsolutePath(), actual2);
     }
 
-    public void testGetFullPath() throws Exception {
-      String hint = "file:///com.android.providers.downloads/test";
+    public void testGenerateSaveFileNoExtension() throws Exception {
+        final File expected = new File(getContext().getFilesDir(), "file.mp4");
+        final String actual = Helpers.generateSaveFile(getContext(),
+                "http://example.com/file", null, null, null,
+                "video/mp4", Downloads.Impl.DESTINATION_CACHE_PARTITION);
+        assertEquals(expected.getAbsolutePath(), actual);
+    }
+
+    public void testGenerateSaveFileHint() throws Exception {
+        final File expected = new File(getContext().getFilesDir(), "meow");
+        final String hint = Uri.fromFile(expected).toString();
 
-      // Test that we never change requested filename.
-      String fileName = Helpers.getFullPath(
-          hint,
-          "video/mp4", // MIME type corresponding to file extension .mp4
-          Downloads.Impl.DESTINATION_FILE_URI,
-          null);
-      assertEquals(hint, fileName);
+        // Test that we never change requested filename.
+        final String actual = Helpers.generateSaveFile(getContext(), "url", hint,
+                "dispo", "locat", "video/mp4", Downloads.Impl.DESTINATION_FILE_URI);
+        assertEquals(expected.getAbsolutePath(), actual);
     }
 
+    public void testGenerateSaveFileDisposition() throws Exception {
+        final File expected = new File(getContext().getFilesDir(), "real.mp4");
+        final String actual = Helpers.generateSaveFile(getContext(),
+                "http://example.com/file.txt", null, "attachment; filename=\"subdir/real.pdf\"",
+                null, "video/mp4", Downloads.Impl.DESTINATION_CACHE_PARTITION);
+        assertEquals(expected.getAbsolutePath(), actual);
+    }
 }
index bde9581..d7b389c 100644 (file)
@@ -53,6 +53,8 @@ import com.google.mockwebserver.MockResponse;
 import com.google.mockwebserver.RecordedRequest;
 import com.google.mockwebserver.SocketPolicy;
 
+import libcore.io.IoUtils;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -83,9 +85,7 @@ public class PublicApiFunctionalTest extends AbstractPublicApiTest {
         mTestDirectory = new File(Environment.getExternalStorageDirectory() + File.separator
                                   + "download_manager_functional_test");
         if (mTestDirectory.exists()) {
-            for (File file : mTestDirectory.listFiles()) {
-                file.delete();
-            }
+            IoUtils.deleteContents(mTestDirectory);
         } else {
             mTestDirectory.mkdir();
         }
@@ -94,9 +94,7 @@ public class PublicApiFunctionalTest extends AbstractPublicApiTest {
     @Override
     protected void tearDown() throws Exception {
         if (mTestDirectory != null && mTestDirectory.exists()) {
-            for (File file : mTestDirectory.listFiles()) {
-                file.delete();
-            }
+            IoUtils.deleteContents(mTestDirectory);
             mTestDirectory.delete();
         }
         super.tearDown();
@@ -223,7 +221,7 @@ public class PublicApiFunctionalTest extends AbstractPublicApiTest {
         boolean isFirstResponse = (start == 0);
         int status = isFirstResponse ? HTTP_OK : HTTP_PARTIAL;
         MockResponse response = buildResponse(status, FILE_CONTENT.substring(start, end))
-                .setHeader("Content-length", totalLength)
+                .setHeader("Content-length", isFirstResponse ? totalLength : (end - start))
                 .setHeader("Etag", ETAG);
         if (!isFirstResponse) {
             response.setHeader(
@@ -475,7 +473,7 @@ public class PublicApiFunctionalTest extends AbstractPublicApiTest {
         // 2. Try resuming A, but fail ETag check
         mSystemFacade.incrementTimeMillis(RETRY_DELAY_MILLIS);
         download.runUntilStatus(STATUS_FAILED);
-        assertEquals(HTTP_PRECON_FAILED, download.getReason());
+        assertEquals(DownloadManager.ERROR_CANNOT_RESUME, download.getReason());
         req = takeRequest();
         assertEquals("bytes=2-", getHeaderValue(req, "Range"));
         assertEquals(A, getHeaderValue(req, "If-Match"));
diff --git a/tests/src/com/android/providers/downloads/StorageTest.java b/tests/src/com/android/providers/downloads/StorageTest.java
new file mode 100644 (file)
index 0000000..eaac3bd
--- /dev/null
@@ -0,0 +1,248 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.providers.downloads;
+
+import static android.app.DownloadManager.COLUMN_REASON;
+import static android.app.DownloadManager.ERROR_INSUFFICIENT_SPACE;
+import static android.app.DownloadManager.STATUS_FAILED;
+import static android.app.DownloadManager.STATUS_SUCCESSFUL;
+import static android.provider.Downloads.Impl.DESTINATION_CACHE_PARTITION;
+import static android.provider.Downloads.Impl.DESTINATION_SYSTEMCACHE_PARTITION;
+
+import android.app.DownloadManager;
+import android.content.pm.PackageManager;
+import android.os.Environment;
+import android.os.StatFs;
+import android.provider.Downloads.Impl;
+import android.test.MoreAsserts;
+import android.util.Log;
+
+import com.android.providers.downloads.StorageUtils.ObserverLatch;
+import com.google.mockwebserver.MockResponse;
+import com.google.mockwebserver.SocketPolicy;
+
+import libcore.io.ErrnoException;
+import libcore.io.ForwardingOs;
+import libcore.io.IoUtils;
+import libcore.io.Libcore;
+import libcore.io.Os;
+import libcore.io.StructStatVfs;
+
+import java.io.File;
+import java.io.FileDescriptor;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public class StorageTest extends AbstractPublicApiTest {
+    private static final String TAG = "StorageTest";
+
+    private static final int DOWNLOAD_SIZE = 512 * 1024;
+    private static final byte[] DOWNLOAD_BODY;
+
+    static {
+        DOWNLOAD_BODY = new byte[DOWNLOAD_SIZE];
+        for (int i = 0; i < DOWNLOAD_SIZE; i++) {
+            DOWNLOAD_BODY[i] = (byte) (i % 32);
+        }
+    }
+
+    private Os mOriginal;
+    private long mStealBytes;
+
+    public StorageTest() {
+        super(new FakeSystemFacade());
+    }
+
+    @Override
+    protected void setUp() throws Exception {
+        super.setUp();
+
+        StorageUtils.sForceFullEviction = true;
+        mStealBytes = 0;
+
+        mOriginal = Libcore.os;
+        Libcore.os = new ForwardingOs(mOriginal) {
+            @Override
+            public StructStatVfs statvfs(String path) throws ErrnoException {
+                return stealBytes(os.statvfs(path));
+            }
+
+            @Override
+            public StructStatVfs fstatvfs(FileDescriptor fd) throws ErrnoException {
+                return stealBytes(os.fstatvfs(fd));
+            }
+
+            private StructStatVfs stealBytes(StructStatVfs s) {
+                final long stealBlocks = (mStealBytes + (s.f_bsize - 1)) / s.f_bsize;
+                final long f_bavail = s.f_bavail - stealBlocks;
+                return new StructStatVfs(s.f_bsize, s.f_frsize, s.f_blocks, s.f_bfree, f_bavail,
+                        s.f_files, s.f_ffree, s.f_favail, s.f_fsid, s.f_flag, s.f_namemax);
+            }
+        };
+    }
+
+    @Override
+    protected void tearDown() throws Exception {
+        super.tearDown();
+
+        StorageUtils.sForceFullEviction = false;
+        mStealBytes = 0;
+
+        if (mOriginal != null) {
+            Libcore.os = mOriginal;
+        }
+    }
+
+    private enum CacheStatus { CLEAN, DIRTY }
+    private enum BodyType { COMPLETE, CHUNKED }
+
+    public void testDataDirtyComplete() throws Exception {
+        prepareAndRunDownload(DESTINATION_CACHE_PARTITION,
+                CacheStatus.DIRTY, BodyType.COMPLETE,
+                STATUS_SUCCESSFUL, -1);
+    }
+
+    public void testDataDirtyChunked() throws Exception {
+        prepareAndRunDownload(DESTINATION_CACHE_PARTITION,
+                CacheStatus.DIRTY, BodyType.CHUNKED,
+                STATUS_SUCCESSFUL, -1);
+    }
+
+    public void testDataCleanComplete() throws Exception {
+        prepareAndRunDownload(DESTINATION_CACHE_PARTITION,
+                CacheStatus.CLEAN, BodyType.COMPLETE,
+                STATUS_FAILED, ERROR_INSUFFICIENT_SPACE);
+    }
+
+    public void testDataCleanChunked() throws Exception {
+        prepareAndRunDownload(DESTINATION_CACHE_PARTITION,
+                CacheStatus.CLEAN, BodyType.CHUNKED,
+                STATUS_FAILED, ERROR_INSUFFICIENT_SPACE);
+    }
+
+    public void testCacheDirtyComplete() throws Exception {
+        prepareAndRunDownload(DESTINATION_SYSTEMCACHE_PARTITION,
+                CacheStatus.DIRTY, BodyType.COMPLETE,
+                STATUS_SUCCESSFUL, -1);
+    }
+
+    public void testCacheDirtyChunked() throws Exception {
+        prepareAndRunDownload(DESTINATION_SYSTEMCACHE_PARTITION,
+                CacheStatus.DIRTY, BodyType.CHUNKED,
+                STATUS_SUCCESSFUL, -1);
+    }
+
+    public void testCacheCleanComplete() throws Exception {
+        prepareAndRunDownload(DESTINATION_SYSTEMCACHE_PARTITION,
+                CacheStatus.CLEAN, BodyType.COMPLETE,
+                STATUS_FAILED, ERROR_INSUFFICIENT_SPACE);
+    }
+
+    public void testCacheCleanChunked() throws Exception {
+        prepareAndRunDownload(DESTINATION_SYSTEMCACHE_PARTITION,
+                CacheStatus.CLEAN, BodyType.CHUNKED,
+                STATUS_FAILED, ERROR_INSUFFICIENT_SPACE);
+    }
+
+    private void prepareAndRunDownload(
+            int dest, CacheStatus cache, BodyType body, int expectedStatus, int expectedReason)
+            throws Exception {
+
+        // Ensure that we've purged everything possible for destination
+        final File dirtyDir;
+        if (dest == DESTINATION_CACHE_PARTITION) {
+            final PackageManager pm = getContext().getPackageManager();
+            final ObserverLatch observer = new ObserverLatch();
+            pm.freeStorageAndNotify(Long.MAX_VALUE, observer);
+
+            try {
+                if (!observer.latch.await(30, TimeUnit.SECONDS)) {
+                    throw new IOException("Timeout while freeing disk space");
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+
+            dirtyDir = getContext().getCacheDir();
+
+        } else if (dest == DESTINATION_SYSTEMCACHE_PARTITION) {
+            IoUtils.deleteContents(Environment.getDownloadCacheDirectory());
+            dirtyDir = Environment.getDownloadCacheDirectory();
+
+        } else {
+            throw new IllegalArgumentException("Unknown destination");
+        }
+
+        // Allocate a cache file, if requested, making it large enough and old
+        // enough to clear.
+        final File dirtyFile;
+        if (cache == CacheStatus.DIRTY) {
+            dirtyFile = new File(dirtyDir, "cache_file.bin");
+            assertTrue(dirtyFile.createNewFile());
+            final FileOutputStream os = new FileOutputStream(dirtyFile);
+            final int dirtySize = (DOWNLOAD_SIZE * 3) / 2;
+            Libcore.os.posix_fallocate(os.getFD(), 0, dirtySize);
+            IoUtils.closeQuietly(os);
+
+            dirtyFile.setLastModified(
+                    System.currentTimeMillis() - (StorageUtils.MIN_DELETE_AGE * 2));
+        } else {
+            dirtyFile = null;
+        }
+
+        // At this point, hide all other disk space to make the download fail;
+        // if we have a dirty cache file it can be cleared to let us proceed.
+        final long targetFree = StorageUtils.RESERVED_BYTES + (DOWNLOAD_SIZE / 2);
+
+        final StatFs stat = new StatFs(dirtyDir.getAbsolutePath());
+        Log.d(TAG, "Available bytes (before steal): " + stat.getAvailableBytes());
+        mStealBytes = stat.getAvailableBytes() - targetFree;
+
+        stat.restat(dirtyDir.getAbsolutePath());
+        Log.d(TAG, "Available bytes (after steal): " + stat.getAvailableBytes());
+
+        final MockResponse resp = new MockResponse().setResponseCode(200)
+                .setHeader("Content-type", "text/plain")
+                .setSocketPolicy(SocketPolicy.DISCONNECT_AT_END);
+        if (body == BodyType.CHUNKED) {
+            resp.setChunkedBody(DOWNLOAD_BODY, 1021);
+        } else {
+            resp.setBody(DOWNLOAD_BODY);
+        }
+        enqueueResponse(resp);
+
+        final DownloadManager.Request req = getRequest();
+        if (dest == Impl.DESTINATION_SYSTEMCACHE_PARTITION) {
+            req.setDestinationToSystemCache();
+        }
+        final Download download = enqueueRequest(req);
+        download.runUntilStatus(expectedStatus);
+
+        if (expectedStatus == STATUS_SUCCESSFUL) {
+            MoreAsserts.assertEquals(DOWNLOAD_BODY, download.getRawContents());
+        }
+
+        if (expectedReason != -1) {
+            assertEquals(expectedReason, download.getLongField(COLUMN_REASON));
+        }
+
+        if (dirtyFile != null) {
+            assertFalse(dirtyFile.exists());
+        }
+    }
+}
index 920f703..1e50144 100644 (file)
@@ -27,6 +27,7 @@ import com.google.android.collect.Sets;
 import com.google.mockwebserver.MockResponse;
 import com.google.mockwebserver.SocketPolicy;
 
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -60,9 +61,10 @@ public class ThreadingTest extends AbstractPublicApiTest {
 
     public void testFilenameRace() throws Exception {
         final List<Pair<Download, String>> downloads = Lists.newArrayList();
+        final HashSet<String> expectedBodies = Sets.newHashSet();
 
         // Request dozen files at once with same name
-        for (int i = 0; i < 12; i++) {
+        for (int i = 0; i < 32; i++) {
             final String body = "DOWNLOAD " + i + " CONTENTS";
             enqueueResponse(new MockResponse().setResponseCode(HTTP_OK).setBody(body)
                     .setHeader("Content-type", "text/plain")
@@ -70,6 +72,7 @@ public class ThreadingTest extends AbstractPublicApiTest {
 
             final Download d = enqueueRequest(getRequest());
             downloads.add(Pair.create(d, body));
+            expectedBodies.add(body);
         }
 
         // Kick off downloads in parallel
@@ -82,6 +85,7 @@ public class ThreadingTest extends AbstractPublicApiTest {
 
         // Ensure that contents are clean and filenames unique
         final Set<String> seenFiles = Sets.newHashSet();
+        final HashSet<String> actualBodies = Sets.newHashSet();
 
         for (Pair<Download, String> d : downloads) {
             final String file = d.first.getStringField(DownloadManager.COLUMN_LOCAL_FILENAME);
@@ -91,7 +95,10 @@ public class ThreadingTest extends AbstractPublicApiTest {
 
             final String expected = d.second;
             final String actual = d.first.getContents();
-            assertEquals(expected, actual);
+
+            actualBodies.add(actual);
         }
+
+        assertEquals(expectedBodies, actualBodies);
     }
 }