FS-Cache: Add counters for entry/exit to/from cache operation functions
[linux-2.6.git] / fs / fscache / page.c
1 /* Cache page management and data I/O routines
2  *
3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
4  * Written by David Howells (dhowells@redhat.com)
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version
9  * 2 of the License, or (at your option) any later version.
10  */
11
12 #define FSCACHE_DEBUG_LEVEL PAGE
13 #include <linux/module.h>
14 #include <linux/fscache-cache.h>
15 #include <linux/buffer_head.h>
16 #include <linux/pagevec.h>
17 #include "internal.h"
18
19 /*
20  * check to see if a page is being written to the cache
21  */
22 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
23 {
24         void *val;
25
26         rcu_read_lock();
27         val = radix_tree_lookup(&cookie->stores, page->index);
28         rcu_read_unlock();
29
30         return val != NULL;
31 }
32 EXPORT_SYMBOL(__fscache_check_page_write);
33
34 /*
35  * wait for a page to finish being written to the cache
36  */
37 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
38 {
39         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
40
41         wait_event(*wq, !__fscache_check_page_write(cookie, page));
42 }
43 EXPORT_SYMBOL(__fscache_wait_on_page_write);
44
45 /*
46  * note that a page has finished being written to the cache
47  */
48 static void fscache_end_page_write(struct fscache_cookie *cookie, struct page *page)
49 {
50         struct page *xpage;
51
52         spin_lock(&cookie->lock);
53         xpage = radix_tree_delete(&cookie->stores, page->index);
54         spin_unlock(&cookie->lock);
55         ASSERT(xpage != NULL);
56
57         wake_up_bit(&cookie->flags, 0);
58 }
59
60 /*
61  * actually apply the changed attributes to a cache object
62  */
63 static void fscache_attr_changed_op(struct fscache_operation *op)
64 {
65         struct fscache_object *object = op->object;
66         int ret;
67
68         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
69
70         fscache_stat(&fscache_n_attr_changed_calls);
71
72         if (fscache_object_is_active(object)) {
73                 fscache_set_op_state(op, "CallFS");
74                 fscache_stat(&fscache_n_cop_attr_changed);
75                 ret = object->cache->ops->attr_changed(object);
76                 fscache_stat_d(&fscache_n_cop_attr_changed);
77                 fscache_set_op_state(op, "Done");
78                 if (ret < 0)
79                         fscache_abort_object(object);
80         }
81
82         _leave("");
83 }
84
85 /*
86  * notification that the attributes on an object have changed
87  */
88 int __fscache_attr_changed(struct fscache_cookie *cookie)
89 {
90         struct fscache_operation *op;
91         struct fscache_object *object;
92
93         _enter("%p", cookie);
94
95         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
96
97         fscache_stat(&fscache_n_attr_changed);
98
99         op = kzalloc(sizeof(*op), GFP_KERNEL);
100         if (!op) {
101                 fscache_stat(&fscache_n_attr_changed_nomem);
102                 _leave(" = -ENOMEM");
103                 return -ENOMEM;
104         }
105
106         fscache_operation_init(op, NULL);
107         fscache_operation_init_slow(op, fscache_attr_changed_op);
108         op->flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_EXCLUSIVE);
109         fscache_set_op_name(op, "Attr");
110
111         spin_lock(&cookie->lock);
112
113         if (hlist_empty(&cookie->backing_objects))
114                 goto nobufs;
115         object = hlist_entry(cookie->backing_objects.first,
116                              struct fscache_object, cookie_link);
117
118         if (fscache_submit_exclusive_op(object, op) < 0)
119                 goto nobufs;
120         spin_unlock(&cookie->lock);
121         fscache_stat(&fscache_n_attr_changed_ok);
122         fscache_put_operation(op);
123         _leave(" = 0");
124         return 0;
125
126 nobufs:
127         spin_unlock(&cookie->lock);
128         kfree(op);
129         fscache_stat(&fscache_n_attr_changed_nobufs);
130         _leave(" = %d", -ENOBUFS);
131         return -ENOBUFS;
132 }
133 EXPORT_SYMBOL(__fscache_attr_changed);
134
135 /*
136  * handle secondary execution given to a retrieval op on behalf of the
137  * cache
138  */
139 static void fscache_retrieval_work(struct work_struct *work)
140 {
141         struct fscache_retrieval *op =
142                 container_of(work, struct fscache_retrieval, op.fast_work);
143         unsigned long start;
144
145         _enter("{OP%x}", op->op.debug_id);
146
147         start = jiffies;
148         op->op.processor(&op->op);
149         fscache_hist(fscache_ops_histogram, start);
150         fscache_put_operation(&op->op);
151 }
152
153 /*
154  * release a retrieval op reference
155  */
156 static void fscache_release_retrieval_op(struct fscache_operation *_op)
157 {
158         struct fscache_retrieval *op =
159                 container_of(_op, struct fscache_retrieval, op);
160
161         _enter("{OP%x}", op->op.debug_id);
162
163         fscache_hist(fscache_retrieval_histogram, op->start_time);
164         if (op->context)
165                 fscache_put_context(op->op.object->cookie, op->context);
166
167         _leave("");
168 }
169
170 /*
171  * allocate a retrieval op
172  */
173 static struct fscache_retrieval *fscache_alloc_retrieval(
174         struct address_space *mapping,
175         fscache_rw_complete_t end_io_func,
176         void *context)
177 {
178         struct fscache_retrieval *op;
179
180         /* allocate a retrieval operation and attempt to submit it */
181         op = kzalloc(sizeof(*op), GFP_NOIO);
182         if (!op) {
183                 fscache_stat(&fscache_n_retrievals_nomem);
184                 return NULL;
185         }
186
187         fscache_operation_init(&op->op, fscache_release_retrieval_op);
188         op->op.flags    = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING);
189         op->mapping     = mapping;
190         op->end_io_func = end_io_func;
191         op->context     = context;
192         op->start_time  = jiffies;
193         INIT_WORK(&op->op.fast_work, fscache_retrieval_work);
194         INIT_LIST_HEAD(&op->to_do);
195         fscache_set_op_name(&op->op, "Retr");
196         return op;
197 }
198
199 /*
200  * wait for a deferred lookup to complete
201  */
202 static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
203 {
204         unsigned long jif;
205
206         _enter("");
207
208         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
209                 _leave(" = 0 [imm]");
210                 return 0;
211         }
212
213         fscache_stat(&fscache_n_retrievals_wait);
214
215         jif = jiffies;
216         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
217                         fscache_wait_bit_interruptible,
218                         TASK_INTERRUPTIBLE) != 0) {
219                 fscache_stat(&fscache_n_retrievals_intr);
220                 _leave(" = -ERESTARTSYS");
221                 return -ERESTARTSYS;
222         }
223
224         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
225
226         smp_rmb();
227         fscache_hist(fscache_retrieval_delay_histogram, jif);
228         _leave(" = 0 [dly]");
229         return 0;
230 }
231
232 /*
233  * read a page from the cache or allocate a block in which to store it
234  * - we return:
235  *   -ENOMEM    - out of memory, nothing done
236  *   -ERESTARTSYS - interrupted
237  *   -ENOBUFS   - no backing object available in which to cache the block
238  *   -ENODATA   - no data available in the backing object for this block
239  *   0          - dispatched a read - it'll call end_io_func() when finished
240  */
241 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
242                                  struct page *page,
243                                  fscache_rw_complete_t end_io_func,
244                                  void *context,
245                                  gfp_t gfp)
246 {
247         struct fscache_retrieval *op;
248         struct fscache_object *object;
249         int ret;
250
251         _enter("%p,%p,,,", cookie, page);
252
253         fscache_stat(&fscache_n_retrievals);
254
255         if (hlist_empty(&cookie->backing_objects))
256                 goto nobufs;
257
258         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
259         ASSERTCMP(page, !=, NULL);
260
261         if (fscache_wait_for_deferred_lookup(cookie) < 0)
262                 return -ERESTARTSYS;
263
264         op = fscache_alloc_retrieval(page->mapping, end_io_func, context);
265         if (!op) {
266                 _leave(" = -ENOMEM");
267                 return -ENOMEM;
268         }
269         fscache_set_op_name(&op->op, "RetrRA1");
270
271         spin_lock(&cookie->lock);
272
273         if (hlist_empty(&cookie->backing_objects))
274                 goto nobufs_unlock;
275         object = hlist_entry(cookie->backing_objects.first,
276                              struct fscache_object, cookie_link);
277
278         ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP);
279
280         atomic_inc(&object->n_reads);
281         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
282
283         if (fscache_submit_op(object, &op->op) < 0)
284                 goto nobufs_unlock;
285         spin_unlock(&cookie->lock);
286
287         fscache_stat(&fscache_n_retrieval_ops);
288
289         /* pin the netfs read context in case we need to do the actual netfs
290          * read because we've encountered a cache read failure */
291         fscache_get_context(object->cookie, op->context);
292
293         /* we wait for the operation to become active, and then process it
294          * *here*, in this thread, and not in the thread pool */
295         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
296                 _debug(">>> WT");
297                 fscache_stat(&fscache_n_retrieval_op_waits);
298                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
299                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
300                 _debug("<<< GO");
301         }
302
303         /* ask the cache to honour the operation */
304         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
305                 fscache_stat(&fscache_n_cop_allocate_page);
306                 ret = object->cache->ops->allocate_page(op, page, gfp);
307                 fscache_stat_d(&fscache_n_cop_allocate_page);
308                 if (ret == 0)
309                         ret = -ENODATA;
310         } else {
311                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
312                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
313                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
314         }
315
316         if (ret == -ENOMEM)
317                 fscache_stat(&fscache_n_retrievals_nomem);
318         else if (ret == -ERESTARTSYS)
319                 fscache_stat(&fscache_n_retrievals_intr);
320         else if (ret == -ENODATA)
321                 fscache_stat(&fscache_n_retrievals_nodata);
322         else if (ret < 0)
323                 fscache_stat(&fscache_n_retrievals_nobufs);
324         else
325                 fscache_stat(&fscache_n_retrievals_ok);
326
327         fscache_put_retrieval(op);
328         _leave(" = %d", ret);
329         return ret;
330
331 nobufs_unlock:
332         spin_unlock(&cookie->lock);
333         kfree(op);
334 nobufs:
335         fscache_stat(&fscache_n_retrievals_nobufs);
336         _leave(" = -ENOBUFS");
337         return -ENOBUFS;
338 }
339 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
340
341 /*
342  * read a list of page from the cache or allocate a block in which to store
343  * them
344  * - we return:
345  *   -ENOMEM    - out of memory, some pages may be being read
346  *   -ERESTARTSYS - interrupted, some pages may be being read
347  *   -ENOBUFS   - no backing object or space available in which to cache any
348  *                pages not being read
349  *   -ENODATA   - no data available in the backing object for some or all of
350  *                the pages
351  *   0          - dispatched a read on all pages
352  *
353  * end_io_func() will be called for each page read from the cache as it is
354  * finishes being read
355  *
356  * any pages for which a read is dispatched will be removed from pages and
357  * nr_pages
358  */
359 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
360                                   struct address_space *mapping,
361                                   struct list_head *pages,
362                                   unsigned *nr_pages,
363                                   fscache_rw_complete_t end_io_func,
364                                   void *context,
365                                   gfp_t gfp)
366 {
367         struct fscache_retrieval *op;
368         struct fscache_object *object;
369         int ret;
370
371         _enter("%p,,%d,,,", cookie, *nr_pages);
372
373         fscache_stat(&fscache_n_retrievals);
374
375         if (hlist_empty(&cookie->backing_objects))
376                 goto nobufs;
377
378         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
379         ASSERTCMP(*nr_pages, >, 0);
380         ASSERT(!list_empty(pages));
381
382         if (fscache_wait_for_deferred_lookup(cookie) < 0)
383                 return -ERESTARTSYS;
384
385         op = fscache_alloc_retrieval(mapping, end_io_func, context);
386         if (!op)
387                 return -ENOMEM;
388         fscache_set_op_name(&op->op, "RetrRAN");
389
390         spin_lock(&cookie->lock);
391
392         if (hlist_empty(&cookie->backing_objects))
393                 goto nobufs_unlock;
394         object = hlist_entry(cookie->backing_objects.first,
395                              struct fscache_object, cookie_link);
396
397         atomic_inc(&object->n_reads);
398         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
399
400         if (fscache_submit_op(object, &op->op) < 0)
401                 goto nobufs_unlock;
402         spin_unlock(&cookie->lock);
403
404         fscache_stat(&fscache_n_retrieval_ops);
405
406         /* pin the netfs read context in case we need to do the actual netfs
407          * read because we've encountered a cache read failure */
408         fscache_get_context(object->cookie, op->context);
409
410         /* we wait for the operation to become active, and then process it
411          * *here*, in this thread, and not in the thread pool */
412         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
413                 _debug(">>> WT");
414                 fscache_stat(&fscache_n_retrieval_op_waits);
415                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
416                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
417                 _debug("<<< GO");
418         }
419
420         /* ask the cache to honour the operation */
421         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
422                 fscache_stat(&fscache_n_cop_allocate_pages);
423                 ret = object->cache->ops->allocate_pages(
424                         op, pages, nr_pages, gfp);
425                 fscache_stat_d(&fscache_n_cop_allocate_pages);
426         } else {
427                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
428                 ret = object->cache->ops->read_or_alloc_pages(
429                         op, pages, nr_pages, gfp);
430                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
431         }
432
433         if (ret == -ENOMEM)
434                 fscache_stat(&fscache_n_retrievals_nomem);
435         else if (ret == -ERESTARTSYS)
436                 fscache_stat(&fscache_n_retrievals_intr);
437         else if (ret == -ENODATA)
438                 fscache_stat(&fscache_n_retrievals_nodata);
439         else if (ret < 0)
440                 fscache_stat(&fscache_n_retrievals_nobufs);
441         else
442                 fscache_stat(&fscache_n_retrievals_ok);
443
444         fscache_put_retrieval(op);
445         _leave(" = %d", ret);
446         return ret;
447
448 nobufs_unlock:
449         spin_unlock(&cookie->lock);
450         kfree(op);
451 nobufs:
452         fscache_stat(&fscache_n_retrievals_nobufs);
453         _leave(" = -ENOBUFS");
454         return -ENOBUFS;
455 }
456 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
457
458 /*
459  * allocate a block in the cache on which to store a page
460  * - we return:
461  *   -ENOMEM    - out of memory, nothing done
462  *   -ERESTARTSYS - interrupted
463  *   -ENOBUFS   - no backing object available in which to cache the block
464  *   0          - block allocated
465  */
466 int __fscache_alloc_page(struct fscache_cookie *cookie,
467                          struct page *page,
468                          gfp_t gfp)
469 {
470         struct fscache_retrieval *op;
471         struct fscache_object *object;
472         int ret;
473
474         _enter("%p,%p,,,", cookie, page);
475
476         fscache_stat(&fscache_n_allocs);
477
478         if (hlist_empty(&cookie->backing_objects))
479                 goto nobufs;
480
481         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
482         ASSERTCMP(page, !=, NULL);
483
484         if (fscache_wait_for_deferred_lookup(cookie) < 0)
485                 return -ERESTARTSYS;
486
487         op = fscache_alloc_retrieval(page->mapping, NULL, NULL);
488         if (!op)
489                 return -ENOMEM;
490         fscache_set_op_name(&op->op, "RetrAL1");
491
492         spin_lock(&cookie->lock);
493
494         if (hlist_empty(&cookie->backing_objects))
495                 goto nobufs_unlock;
496         object = hlist_entry(cookie->backing_objects.first,
497                              struct fscache_object, cookie_link);
498
499         if (fscache_submit_op(object, &op->op) < 0)
500                 goto nobufs_unlock;
501         spin_unlock(&cookie->lock);
502
503         fscache_stat(&fscache_n_alloc_ops);
504
505         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
506                 _debug(">>> WT");
507                 fscache_stat(&fscache_n_alloc_op_waits);
508                 wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
509                             fscache_wait_bit, TASK_UNINTERRUPTIBLE);
510                 _debug("<<< GO");
511         }
512
513         /* ask the cache to honour the operation */
514         fscache_stat(&fscache_n_cop_allocate_page);
515         ret = object->cache->ops->allocate_page(op, page, gfp);
516         fscache_stat_d(&fscache_n_cop_allocate_page);
517
518         if (ret < 0)
519                 fscache_stat(&fscache_n_allocs_nobufs);
520         else
521                 fscache_stat(&fscache_n_allocs_ok);
522
523         fscache_put_retrieval(op);
524         _leave(" = %d", ret);
525         return ret;
526
527 nobufs_unlock:
528         spin_unlock(&cookie->lock);
529         kfree(op);
530 nobufs:
531         fscache_stat(&fscache_n_allocs_nobufs);
532         _leave(" = -ENOBUFS");
533         return -ENOBUFS;
534 }
535 EXPORT_SYMBOL(__fscache_alloc_page);
536
537 /*
538  * release a write op reference
539  */
540 static void fscache_release_write_op(struct fscache_operation *_op)
541 {
542         _enter("{OP%x}", _op->debug_id);
543 }
544
545 /*
546  * perform the background storage of a page into the cache
547  */
548 static void fscache_write_op(struct fscache_operation *_op)
549 {
550         struct fscache_storage *op =
551                 container_of(_op, struct fscache_storage, op);
552         struct fscache_object *object = op->op.object;
553         struct fscache_cookie *cookie = object->cookie;
554         struct page *page;
555         unsigned n;
556         void *results[1];
557         int ret;
558
559         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
560
561         fscache_set_op_state(&op->op, "GetPage");
562
563         spin_lock(&cookie->lock);
564         spin_lock(&object->lock);
565
566         if (!fscache_object_is_active(object)) {
567                 spin_unlock(&object->lock);
568                 spin_unlock(&cookie->lock);
569                 _leave("");
570                 return;
571         }
572
573         fscache_stat(&fscache_n_store_calls);
574
575         /* find a page to store */
576         page = NULL;
577         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
578                                        FSCACHE_COOKIE_PENDING_TAG);
579         if (n != 1)
580                 goto superseded;
581         page = results[0];
582         _debug("gang %d [%lx]", n, page->index);
583         if (page->index > op->store_limit)
584                 goto superseded;
585
586         radix_tree_tag_clear(&cookie->stores, page->index,
587                              FSCACHE_COOKIE_PENDING_TAG);
588
589         spin_unlock(&object->lock);
590         spin_unlock(&cookie->lock);
591
592         if (page) {
593                 fscache_set_op_state(&op->op, "Store");
594                 fscache_stat(&fscache_n_cop_write_page);
595                 ret = object->cache->ops->write_page(op, page);
596                 fscache_stat_d(&fscache_n_cop_write_page);
597                 fscache_set_op_state(&op->op, "EndWrite");
598                 fscache_end_page_write(cookie, page);
599                 page_cache_release(page);
600                 if (ret < 0) {
601                         fscache_set_op_state(&op->op, "Abort");
602                         fscache_abort_object(object);
603                 } else {
604                         fscache_enqueue_operation(&op->op);
605                 }
606         }
607
608         _leave("");
609         return;
610
611 superseded:
612         /* this writer is going away and there aren't any more things to
613          * write */
614         _debug("cease");
615         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
616         spin_unlock(&object->lock);
617         spin_unlock(&cookie->lock);
618         _leave("");
619 }
620
621 /*
622  * request a page be stored in the cache
623  * - returns:
624  *   -ENOMEM    - out of memory, nothing done
625  *   -ENOBUFS   - no backing object available in which to cache the page
626  *   0          - dispatched a write - it'll call end_io_func() when finished
627  *
628  * if the cookie still has a backing object at this point, that object can be
629  * in one of a few states with respect to storage processing:
630  *
631  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
632  *      set)
633  *
634  *      (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred
635  *          fill op)
636  *
637  *      (b) writes deferred till post-creation (mark page for writing and
638  *          return immediately)
639  *
640  *  (2) negative lookup, object created, initial fill being made from netfs
641  *      (FSCACHE_COOKIE_INITIAL_FILL is set)
642  *
643  *      (a) fill point not yet reached this page (mark page for writing and
644  *          return)
645  *
646  *      (b) fill point passed this page (queue op to store this page)
647  *
648  *  (3) object extant (queue op to store this page)
649  *
650  * any other state is invalid
651  */
652 int __fscache_write_page(struct fscache_cookie *cookie,
653                          struct page *page,
654                          gfp_t gfp)
655 {
656         struct fscache_storage *op;
657         struct fscache_object *object;
658         int ret;
659
660         _enter("%p,%x,", cookie, (u32) page->flags);
661
662         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
663         ASSERT(PageFsCache(page));
664
665         fscache_stat(&fscache_n_stores);
666
667         op = kzalloc(sizeof(*op), GFP_NOIO);
668         if (!op)
669                 goto nomem;
670
671         fscache_operation_init(&op->op, fscache_release_write_op);
672         fscache_operation_init_slow(&op->op, fscache_write_op);
673         op->op.flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_WAITING);
674         fscache_set_op_name(&op->op, "Write1");
675
676         ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
677         if (ret < 0)
678                 goto nomem_free;
679
680         ret = -ENOBUFS;
681         spin_lock(&cookie->lock);
682
683         if (hlist_empty(&cookie->backing_objects))
684                 goto nobufs;
685         object = hlist_entry(cookie->backing_objects.first,
686                              struct fscache_object, cookie_link);
687         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
688                 goto nobufs;
689
690         /* add the page to the pending-storage radix tree on the backing
691          * object */
692         spin_lock(&object->lock);
693
694         _debug("store limit %llx", (unsigned long long) object->store_limit);
695
696         ret = radix_tree_insert(&cookie->stores, page->index, page);
697         if (ret < 0) {
698                 if (ret == -EEXIST)
699                         goto already_queued;
700                 _debug("insert failed %d", ret);
701                 goto nobufs_unlock_obj;
702         }
703
704         radix_tree_tag_set(&cookie->stores, page->index,
705                            FSCACHE_COOKIE_PENDING_TAG);
706         page_cache_get(page);
707
708         /* we only want one writer at a time, but we do need to queue new
709          * writers after exclusive ops */
710         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
711                 goto already_pending;
712
713         spin_unlock(&object->lock);
714
715         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
716         op->store_limit = object->store_limit;
717
718         if (fscache_submit_op(object, &op->op) < 0)
719                 goto submit_failed;
720
721         spin_unlock(&cookie->lock);
722         radix_tree_preload_end();
723         fscache_stat(&fscache_n_store_ops);
724         fscache_stat(&fscache_n_stores_ok);
725
726         /* the slow work queue now carries its own ref on the object */
727         fscache_put_operation(&op->op);
728         _leave(" = 0");
729         return 0;
730
731 already_queued:
732         fscache_stat(&fscache_n_stores_again);
733 already_pending:
734         spin_unlock(&object->lock);
735         spin_unlock(&cookie->lock);
736         radix_tree_preload_end();
737         kfree(op);
738         fscache_stat(&fscache_n_stores_ok);
739         _leave(" = 0");
740         return 0;
741
742 submit_failed:
743         radix_tree_delete(&cookie->stores, page->index);
744         page_cache_release(page);
745         ret = -ENOBUFS;
746         goto nobufs;
747
748 nobufs_unlock_obj:
749         spin_unlock(&object->lock);
750 nobufs:
751         spin_unlock(&cookie->lock);
752         radix_tree_preload_end();
753         kfree(op);
754         fscache_stat(&fscache_n_stores_nobufs);
755         _leave(" = -ENOBUFS");
756         return -ENOBUFS;
757
758 nomem_free:
759         kfree(op);
760 nomem:
761         fscache_stat(&fscache_n_stores_oom);
762         _leave(" = -ENOMEM");
763         return -ENOMEM;
764 }
765 EXPORT_SYMBOL(__fscache_write_page);
766
767 /*
768  * remove a page from the cache
769  */
770 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
771 {
772         struct fscache_object *object;
773
774         _enter(",%p", page);
775
776         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
777         ASSERTCMP(page, !=, NULL);
778
779         fscache_stat(&fscache_n_uncaches);
780
781         /* cache withdrawal may beat us to it */
782         if (!PageFsCache(page))
783                 goto done;
784
785         /* get the object */
786         spin_lock(&cookie->lock);
787
788         if (hlist_empty(&cookie->backing_objects)) {
789                 ClearPageFsCache(page);
790                 goto done_unlock;
791         }
792
793         object = hlist_entry(cookie->backing_objects.first,
794                              struct fscache_object, cookie_link);
795
796         /* there might now be stuff on disk we could read */
797         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
798
799         /* only invoke the cache backend if we managed to mark the page
800          * uncached here; this deals with synchronisation vs withdrawal */
801         if (TestClearPageFsCache(page) &&
802             object->cache->ops->uncache_page) {
803                 /* the cache backend releases the cookie lock */
804                 fscache_stat(&fscache_n_cop_uncache_page);
805                 object->cache->ops->uncache_page(object, page);
806                 fscache_stat_d(&fscache_n_cop_uncache_page);
807                 goto done;
808         }
809
810 done_unlock:
811         spin_unlock(&cookie->lock);
812 done:
813         _leave("");
814 }
815 EXPORT_SYMBOL(__fscache_uncache_page);
816
817 /**
818  * fscache_mark_pages_cached - Mark pages as being cached
819  * @op: The retrieval op pages are being marked for
820  * @pagevec: The pages to be marked
821  *
822  * Mark a bunch of netfs pages as being cached.  After this is called,
823  * the netfs must call fscache_uncache_page() to remove the mark.
824  */
825 void fscache_mark_pages_cached(struct fscache_retrieval *op,
826                                struct pagevec *pagevec)
827 {
828         struct fscache_cookie *cookie = op->op.object->cookie;
829         unsigned long loop;
830
831 #ifdef CONFIG_FSCACHE_STATS
832         atomic_add(pagevec->nr, &fscache_n_marks);
833 #endif
834
835         for (loop = 0; loop < pagevec->nr; loop++) {
836                 struct page *page = pagevec->pages[loop];
837
838                 _debug("- mark %p{%lx}", page, page->index);
839                 if (TestSetPageFsCache(page)) {
840                         static bool once_only;
841                         if (!once_only) {
842                                 once_only = true;
843                                 printk(KERN_WARNING "FS-Cache:"
844                                        " Cookie type %s marked page %lx"
845                                        " multiple times\n",
846                                        cookie->def->name, page->index);
847                         }
848                 }
849         }
850
851         if (cookie->def->mark_pages_cached)
852                 cookie->def->mark_pages_cached(cookie->netfs_data,
853                                                op->mapping, pagevec);
854         pagevec_reinit(pagevec);
855 }
856 EXPORT_SYMBOL(fscache_mark_pages_cached);