FS-Cache: Fix lock misorder in fscache_write_op()
[linux-2.6.git] / fs / fscache / page.c
1 /* Cache page management and data I/O routines
2  *
3  * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
4  * Written by David Howells (dhowells@redhat.com)
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version
9  * 2 of the License, or (at your option) any later version.
10  */
11
12 #define FSCACHE_DEBUG_LEVEL PAGE
13 #include <linux/module.h>
14 #include <linux/fscache-cache.h>
15 #include <linux/buffer_head.h>
16 #include <linux/pagevec.h>
17 #include "internal.h"
18
19 /*
20  * check to see if a page is being written to the cache
21  */
22 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
23 {
24         void *val;
25
26         rcu_read_lock();
27         val = radix_tree_lookup(&cookie->stores, page->index);
28         rcu_read_unlock();
29
30         return val != NULL;
31 }
32 EXPORT_SYMBOL(__fscache_check_page_write);
33
34 /*
35  * wait for a page to finish being written to the cache
36  */
37 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
38 {
39         wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
40
41         wait_event(*wq, !__fscache_check_page_write(cookie, page));
42 }
43 EXPORT_SYMBOL(__fscache_wait_on_page_write);
44
45 /*
46  * note that a page has finished being written to the cache
47  */
48 static void fscache_end_page_write(struct fscache_object *object,
49                                    struct page *page)
50 {
51         struct fscache_cookie *cookie;
52         struct page *xpage = NULL;
53
54         spin_lock(&object->lock);
55         cookie = object->cookie;
56         if (cookie) {
57                 /* delete the page from the tree if it is now no longer
58                  * pending */
59                 spin_lock(&cookie->stores_lock);
60                 fscache_stat(&fscache_n_store_radix_deletes);
61                 xpage = radix_tree_delete(&cookie->stores, page->index);
62                 spin_unlock(&cookie->stores_lock);
63                 wake_up_bit(&cookie->flags, 0);
64         }
65         spin_unlock(&object->lock);
66         if (xpage)
67                 page_cache_release(xpage);
68 }
69
70 /*
71  * actually apply the changed attributes to a cache object
72  */
73 static void fscache_attr_changed_op(struct fscache_operation *op)
74 {
75         struct fscache_object *object = op->object;
76         int ret;
77
78         _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
79
80         fscache_stat(&fscache_n_attr_changed_calls);
81
82         if (fscache_object_is_active(object)) {
83                 fscache_set_op_state(op, "CallFS");
84                 fscache_stat(&fscache_n_cop_attr_changed);
85                 ret = object->cache->ops->attr_changed(object);
86                 fscache_stat_d(&fscache_n_cop_attr_changed);
87                 fscache_set_op_state(op, "Done");
88                 if (ret < 0)
89                         fscache_abort_object(object);
90         }
91
92         _leave("");
93 }
94
95 /*
96  * notification that the attributes on an object have changed
97  */
98 int __fscache_attr_changed(struct fscache_cookie *cookie)
99 {
100         struct fscache_operation *op;
101         struct fscache_object *object;
102
103         _enter("%p", cookie);
104
105         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
106
107         fscache_stat(&fscache_n_attr_changed);
108
109         op = kzalloc(sizeof(*op), GFP_KERNEL);
110         if (!op) {
111                 fscache_stat(&fscache_n_attr_changed_nomem);
112                 _leave(" = -ENOMEM");
113                 return -ENOMEM;
114         }
115
116         fscache_operation_init(op, NULL);
117         fscache_operation_init_slow(op, fscache_attr_changed_op);
118         op->flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_EXCLUSIVE);
119         fscache_set_op_name(op, "Attr");
120
121         spin_lock(&cookie->lock);
122
123         if (hlist_empty(&cookie->backing_objects))
124                 goto nobufs;
125         object = hlist_entry(cookie->backing_objects.first,
126                              struct fscache_object, cookie_link);
127
128         if (fscache_submit_exclusive_op(object, op) < 0)
129                 goto nobufs;
130         spin_unlock(&cookie->lock);
131         fscache_stat(&fscache_n_attr_changed_ok);
132         fscache_put_operation(op);
133         _leave(" = 0");
134         return 0;
135
136 nobufs:
137         spin_unlock(&cookie->lock);
138         kfree(op);
139         fscache_stat(&fscache_n_attr_changed_nobufs);
140         _leave(" = %d", -ENOBUFS);
141         return -ENOBUFS;
142 }
143 EXPORT_SYMBOL(__fscache_attr_changed);
144
145 /*
146  * handle secondary execution given to a retrieval op on behalf of the
147  * cache
148  */
149 static void fscache_retrieval_work(struct work_struct *work)
150 {
151         struct fscache_retrieval *op =
152                 container_of(work, struct fscache_retrieval, op.fast_work);
153         unsigned long start;
154
155         _enter("{OP%x}", op->op.debug_id);
156
157         start = jiffies;
158         op->op.processor(&op->op);
159         fscache_hist(fscache_ops_histogram, start);
160         fscache_put_operation(&op->op);
161 }
162
163 /*
164  * release a retrieval op reference
165  */
166 static void fscache_release_retrieval_op(struct fscache_operation *_op)
167 {
168         struct fscache_retrieval *op =
169                 container_of(_op, struct fscache_retrieval, op);
170
171         _enter("{OP%x}", op->op.debug_id);
172
173         fscache_hist(fscache_retrieval_histogram, op->start_time);
174         if (op->context)
175                 fscache_put_context(op->op.object->cookie, op->context);
176
177         _leave("");
178 }
179
180 /*
181  * allocate a retrieval op
182  */
183 static struct fscache_retrieval *fscache_alloc_retrieval(
184         struct address_space *mapping,
185         fscache_rw_complete_t end_io_func,
186         void *context)
187 {
188         struct fscache_retrieval *op;
189
190         /* allocate a retrieval operation and attempt to submit it */
191         op = kzalloc(sizeof(*op), GFP_NOIO);
192         if (!op) {
193                 fscache_stat(&fscache_n_retrievals_nomem);
194                 return NULL;
195         }
196
197         fscache_operation_init(&op->op, fscache_release_retrieval_op);
198         op->op.flags    = FSCACHE_OP_MYTHREAD | (1 << FSCACHE_OP_WAITING);
199         op->mapping     = mapping;
200         op->end_io_func = end_io_func;
201         op->context     = context;
202         op->start_time  = jiffies;
203         INIT_WORK(&op->op.fast_work, fscache_retrieval_work);
204         INIT_LIST_HEAD(&op->to_do);
205         fscache_set_op_name(&op->op, "Retr");
206         return op;
207 }
208
209 /*
210  * wait for a deferred lookup to complete
211  */
212 static int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
213 {
214         unsigned long jif;
215
216         _enter("");
217
218         if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
219                 _leave(" = 0 [imm]");
220                 return 0;
221         }
222
223         fscache_stat(&fscache_n_retrievals_wait);
224
225         jif = jiffies;
226         if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
227                         fscache_wait_bit_interruptible,
228                         TASK_INTERRUPTIBLE) != 0) {
229                 fscache_stat(&fscache_n_retrievals_intr);
230                 _leave(" = -ERESTARTSYS");
231                 return -ERESTARTSYS;
232         }
233
234         ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
235
236         smp_rmb();
237         fscache_hist(fscache_retrieval_delay_histogram, jif);
238         _leave(" = 0 [dly]");
239         return 0;
240 }
241
242 /*
243  * read a page from the cache or allocate a block in which to store it
244  * - we return:
245  *   -ENOMEM    - out of memory, nothing done
246  *   -ERESTARTSYS - interrupted
247  *   -ENOBUFS   - no backing object available in which to cache the block
248  *   -ENODATA   - no data available in the backing object for this block
249  *   0          - dispatched a read - it'll call end_io_func() when finished
250  */
251 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
252                                  struct page *page,
253                                  fscache_rw_complete_t end_io_func,
254                                  void *context,
255                                  gfp_t gfp)
256 {
257         struct fscache_retrieval *op;
258         struct fscache_object *object;
259         int ret;
260
261         _enter("%p,%p,,,", cookie, page);
262
263         fscache_stat(&fscache_n_retrievals);
264
265         if (hlist_empty(&cookie->backing_objects))
266                 goto nobufs;
267
268         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
269         ASSERTCMP(page, !=, NULL);
270
271         if (fscache_wait_for_deferred_lookup(cookie) < 0)
272                 return -ERESTARTSYS;
273
274         op = fscache_alloc_retrieval(page->mapping, end_io_func, context);
275         if (!op) {
276                 _leave(" = -ENOMEM");
277                 return -ENOMEM;
278         }
279         fscache_set_op_name(&op->op, "RetrRA1");
280
281         spin_lock(&cookie->lock);
282
283         if (hlist_empty(&cookie->backing_objects))
284                 goto nobufs_unlock;
285         object = hlist_entry(cookie->backing_objects.first,
286                              struct fscache_object, cookie_link);
287
288         ASSERTCMP(object->state, >, FSCACHE_OBJECT_LOOKING_UP);
289
290         atomic_inc(&object->n_reads);
291         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
292
293         if (fscache_submit_op(object, &op->op) < 0)
294                 goto nobufs_unlock;
295         spin_unlock(&cookie->lock);
296
297         fscache_stat(&fscache_n_retrieval_ops);
298
299         /* pin the netfs read context in case we need to do the actual netfs
300          * read because we've encountered a cache read failure */
301         fscache_get_context(object->cookie, op->context);
302
303         /* we wait for the operation to become active, and then process it
304          * *here*, in this thread, and not in the thread pool */
305         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
306                 _debug(">>> WT");
307                 fscache_stat(&fscache_n_retrieval_op_waits);
308                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
309                                 fscache_wait_bit_interruptible,
310                                 TASK_INTERRUPTIBLE) < 0) {
311                         ret = fscache_cancel_op(&op->op);
312                         if (ret == 0) {
313                                 ret = -ERESTARTSYS;
314                                 goto error;
315                         }
316
317                         /* it's been removed from the pending queue by another
318                          * party, so we should get to run shortly */
319                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
320                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
321                 }
322                 _debug("<<< GO");
323         }
324
325         /* ask the cache to honour the operation */
326         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
327                 fscache_stat(&fscache_n_cop_allocate_page);
328                 ret = object->cache->ops->allocate_page(op, page, gfp);
329                 fscache_stat_d(&fscache_n_cop_allocate_page);
330                 if (ret == 0)
331                         ret = -ENODATA;
332         } else {
333                 fscache_stat(&fscache_n_cop_read_or_alloc_page);
334                 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
335                 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
336         }
337
338 error:
339         if (ret == -ENOMEM)
340                 fscache_stat(&fscache_n_retrievals_nomem);
341         else if (ret == -ERESTARTSYS)
342                 fscache_stat(&fscache_n_retrievals_intr);
343         else if (ret == -ENODATA)
344                 fscache_stat(&fscache_n_retrievals_nodata);
345         else if (ret < 0)
346                 fscache_stat(&fscache_n_retrievals_nobufs);
347         else
348                 fscache_stat(&fscache_n_retrievals_ok);
349
350         fscache_put_retrieval(op);
351         _leave(" = %d", ret);
352         return ret;
353
354 nobufs_unlock:
355         spin_unlock(&cookie->lock);
356         kfree(op);
357 nobufs:
358         fscache_stat(&fscache_n_retrievals_nobufs);
359         _leave(" = -ENOBUFS");
360         return -ENOBUFS;
361 }
362 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
363
364 /*
365  * read a list of page from the cache or allocate a block in which to store
366  * them
367  * - we return:
368  *   -ENOMEM    - out of memory, some pages may be being read
369  *   -ERESTARTSYS - interrupted, some pages may be being read
370  *   -ENOBUFS   - no backing object or space available in which to cache any
371  *                pages not being read
372  *   -ENODATA   - no data available in the backing object for some or all of
373  *                the pages
374  *   0          - dispatched a read on all pages
375  *
376  * end_io_func() will be called for each page read from the cache as it is
377  * finishes being read
378  *
379  * any pages for which a read is dispatched will be removed from pages and
380  * nr_pages
381  */
382 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
383                                   struct address_space *mapping,
384                                   struct list_head *pages,
385                                   unsigned *nr_pages,
386                                   fscache_rw_complete_t end_io_func,
387                                   void *context,
388                                   gfp_t gfp)
389 {
390         struct fscache_retrieval *op;
391         struct fscache_object *object;
392         int ret;
393
394         _enter("%p,,%d,,,", cookie, *nr_pages);
395
396         fscache_stat(&fscache_n_retrievals);
397
398         if (hlist_empty(&cookie->backing_objects))
399                 goto nobufs;
400
401         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
402         ASSERTCMP(*nr_pages, >, 0);
403         ASSERT(!list_empty(pages));
404
405         if (fscache_wait_for_deferred_lookup(cookie) < 0)
406                 return -ERESTARTSYS;
407
408         op = fscache_alloc_retrieval(mapping, end_io_func, context);
409         if (!op)
410                 return -ENOMEM;
411         fscache_set_op_name(&op->op, "RetrRAN");
412
413         spin_lock(&cookie->lock);
414
415         if (hlist_empty(&cookie->backing_objects))
416                 goto nobufs_unlock;
417         object = hlist_entry(cookie->backing_objects.first,
418                              struct fscache_object, cookie_link);
419
420         atomic_inc(&object->n_reads);
421         set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
422
423         if (fscache_submit_op(object, &op->op) < 0)
424                 goto nobufs_unlock;
425         spin_unlock(&cookie->lock);
426
427         fscache_stat(&fscache_n_retrieval_ops);
428
429         /* pin the netfs read context in case we need to do the actual netfs
430          * read because we've encountered a cache read failure */
431         fscache_get_context(object->cookie, op->context);
432
433         /* we wait for the operation to become active, and then process it
434          * *here*, in this thread, and not in the thread pool */
435         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
436                 _debug(">>> WT");
437                 fscache_stat(&fscache_n_retrieval_op_waits);
438                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
439                                 fscache_wait_bit_interruptible,
440                                 TASK_INTERRUPTIBLE) < 0) {
441                         ret = fscache_cancel_op(&op->op);
442                         if (ret == 0) {
443                                 ret = -ERESTARTSYS;
444                                 goto error;
445                         }
446
447                         /* it's been removed from the pending queue by another
448                          * party, so we should get to run shortly */
449                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
450                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
451                 }
452                 _debug("<<< GO");
453         }
454
455         /* ask the cache to honour the operation */
456         if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
457                 fscache_stat(&fscache_n_cop_allocate_pages);
458                 ret = object->cache->ops->allocate_pages(
459                         op, pages, nr_pages, gfp);
460                 fscache_stat_d(&fscache_n_cop_allocate_pages);
461         } else {
462                 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
463                 ret = object->cache->ops->read_or_alloc_pages(
464                         op, pages, nr_pages, gfp);
465                 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
466         }
467
468 error:
469         if (ret == -ENOMEM)
470                 fscache_stat(&fscache_n_retrievals_nomem);
471         else if (ret == -ERESTARTSYS)
472                 fscache_stat(&fscache_n_retrievals_intr);
473         else if (ret == -ENODATA)
474                 fscache_stat(&fscache_n_retrievals_nodata);
475         else if (ret < 0)
476                 fscache_stat(&fscache_n_retrievals_nobufs);
477         else
478                 fscache_stat(&fscache_n_retrievals_ok);
479
480         fscache_put_retrieval(op);
481         _leave(" = %d", ret);
482         return ret;
483
484 nobufs_unlock:
485         spin_unlock(&cookie->lock);
486         kfree(op);
487 nobufs:
488         fscache_stat(&fscache_n_retrievals_nobufs);
489         _leave(" = -ENOBUFS");
490         return -ENOBUFS;
491 }
492 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
493
494 /*
495  * allocate a block in the cache on which to store a page
496  * - we return:
497  *   -ENOMEM    - out of memory, nothing done
498  *   -ERESTARTSYS - interrupted
499  *   -ENOBUFS   - no backing object available in which to cache the block
500  *   0          - block allocated
501  */
502 int __fscache_alloc_page(struct fscache_cookie *cookie,
503                          struct page *page,
504                          gfp_t gfp)
505 {
506         struct fscache_retrieval *op;
507         struct fscache_object *object;
508         int ret;
509
510         _enter("%p,%p,,,", cookie, page);
511
512         fscache_stat(&fscache_n_allocs);
513
514         if (hlist_empty(&cookie->backing_objects))
515                 goto nobufs;
516
517         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
518         ASSERTCMP(page, !=, NULL);
519
520         if (fscache_wait_for_deferred_lookup(cookie) < 0)
521                 return -ERESTARTSYS;
522
523         op = fscache_alloc_retrieval(page->mapping, NULL, NULL);
524         if (!op)
525                 return -ENOMEM;
526         fscache_set_op_name(&op->op, "RetrAL1");
527
528         spin_lock(&cookie->lock);
529
530         if (hlist_empty(&cookie->backing_objects))
531                 goto nobufs_unlock;
532         object = hlist_entry(cookie->backing_objects.first,
533                              struct fscache_object, cookie_link);
534
535         if (fscache_submit_op(object, &op->op) < 0)
536                 goto nobufs_unlock;
537         spin_unlock(&cookie->lock);
538
539         fscache_stat(&fscache_n_alloc_ops);
540
541         if (test_bit(FSCACHE_OP_WAITING, &op->op.flags)) {
542                 _debug(">>> WT");
543                 fscache_stat(&fscache_n_alloc_op_waits);
544                 if (wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
545                                 fscache_wait_bit_interruptible,
546                                 TASK_INTERRUPTIBLE) < 0) {
547                         ret = fscache_cancel_op(&op->op);
548                         if (ret == 0) {
549                                 ret = -ERESTARTSYS;
550                                 goto error;
551                         }
552
553                         /* it's been removed from the pending queue by another
554                          * party, so we should get to run shortly */
555                         wait_on_bit(&op->op.flags, FSCACHE_OP_WAITING,
556                                     fscache_wait_bit, TASK_UNINTERRUPTIBLE);
557                 }
558                 _debug("<<< GO");
559         }
560
561         /* ask the cache to honour the operation */
562         fscache_stat(&fscache_n_cop_allocate_page);
563         ret = object->cache->ops->allocate_page(op, page, gfp);
564         fscache_stat_d(&fscache_n_cop_allocate_page);
565
566 error:
567         if (ret == -ERESTARTSYS)
568                 fscache_stat(&fscache_n_allocs_intr);
569         else if (ret < 0)
570                 fscache_stat(&fscache_n_allocs_nobufs);
571         else
572                 fscache_stat(&fscache_n_allocs_ok);
573
574         fscache_put_retrieval(op);
575         _leave(" = %d", ret);
576         return ret;
577
578 nobufs_unlock:
579         spin_unlock(&cookie->lock);
580         kfree(op);
581 nobufs:
582         fscache_stat(&fscache_n_allocs_nobufs);
583         _leave(" = -ENOBUFS");
584         return -ENOBUFS;
585 }
586 EXPORT_SYMBOL(__fscache_alloc_page);
587
588 /*
589  * release a write op reference
590  */
591 static void fscache_release_write_op(struct fscache_operation *_op)
592 {
593         _enter("{OP%x}", _op->debug_id);
594 }
595
596 /*
597  * perform the background storage of a page into the cache
598  */
599 static void fscache_write_op(struct fscache_operation *_op)
600 {
601         struct fscache_storage *op =
602                 container_of(_op, struct fscache_storage, op);
603         struct fscache_object *object = op->op.object;
604         struct fscache_cookie *cookie;
605         struct page *page;
606         unsigned n;
607         void *results[1];
608         int ret;
609
610         _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
611
612         fscache_set_op_state(&op->op, "GetPage");
613
614         spin_lock(&object->lock);
615         cookie = object->cookie;
616
617         if (!fscache_object_is_active(object) || !cookie) {
618                 spin_unlock(&object->lock);
619                 _leave("");
620                 return;
621         }
622
623         spin_lock(&cookie->stores_lock);
624
625         fscache_stat(&fscache_n_store_calls);
626
627         /* find a page to store */
628         page = NULL;
629         n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
630                                        FSCACHE_COOKIE_PENDING_TAG);
631         if (n != 1)
632                 goto superseded;
633         page = results[0];
634         _debug("gang %d [%lx]", n, page->index);
635         if (page->index > op->store_limit) {
636                 fscache_stat(&fscache_n_store_pages_over_limit);
637                 goto superseded;
638         }
639
640         radix_tree_tag_clear(&cookie->stores, page->index,
641                              FSCACHE_COOKIE_PENDING_TAG);
642
643         spin_unlock(&cookie->stores_lock);
644         spin_unlock(&object->lock);
645
646         if (page) {
647                 fscache_set_op_state(&op->op, "Store");
648                 fscache_stat(&fscache_n_store_pages);
649                 fscache_stat(&fscache_n_cop_write_page);
650                 ret = object->cache->ops->write_page(op, page);
651                 fscache_stat_d(&fscache_n_cop_write_page);
652                 fscache_set_op_state(&op->op, "EndWrite");
653                 fscache_end_page_write(object, page);
654                 if (ret < 0) {
655                         fscache_set_op_state(&op->op, "Abort");
656                         fscache_abort_object(object);
657                 } else {
658                         fscache_enqueue_operation(&op->op);
659                 }
660         }
661
662         _leave("");
663         return;
664
665 superseded:
666         /* this writer is going away and there aren't any more things to
667          * write */
668         _debug("cease");
669         spin_unlock(&cookie->stores_lock);
670         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
671         spin_unlock(&object->lock);
672         _leave("");
673 }
674
675 /*
676  * request a page be stored in the cache
677  * - returns:
678  *   -ENOMEM    - out of memory, nothing done
679  *   -ENOBUFS   - no backing object available in which to cache the page
680  *   0          - dispatched a write - it'll call end_io_func() when finished
681  *
682  * if the cookie still has a backing object at this point, that object can be
683  * in one of a few states with respect to storage processing:
684  *
685  *  (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
686  *      set)
687  *
688  *      (a) no writes yet (set FSCACHE_COOKIE_PENDING_FILL and queue deferred
689  *          fill op)
690  *
691  *      (b) writes deferred till post-creation (mark page for writing and
692  *          return immediately)
693  *
694  *  (2) negative lookup, object created, initial fill being made from netfs
695  *      (FSCACHE_COOKIE_INITIAL_FILL is set)
696  *
697  *      (a) fill point not yet reached this page (mark page for writing and
698  *          return)
699  *
700  *      (b) fill point passed this page (queue op to store this page)
701  *
702  *  (3) object extant (queue op to store this page)
703  *
704  * any other state is invalid
705  */
706 int __fscache_write_page(struct fscache_cookie *cookie,
707                          struct page *page,
708                          gfp_t gfp)
709 {
710         struct fscache_storage *op;
711         struct fscache_object *object;
712         int ret;
713
714         _enter("%p,%x,", cookie, (u32) page->flags);
715
716         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
717         ASSERT(PageFsCache(page));
718
719         fscache_stat(&fscache_n_stores);
720
721         op = kzalloc(sizeof(*op), GFP_NOIO);
722         if (!op)
723                 goto nomem;
724
725         fscache_operation_init(&op->op, fscache_release_write_op);
726         fscache_operation_init_slow(&op->op, fscache_write_op);
727         op->op.flags = FSCACHE_OP_SLOW | (1 << FSCACHE_OP_WAITING);
728         fscache_set_op_name(&op->op, "Write1");
729
730         ret = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
731         if (ret < 0)
732                 goto nomem_free;
733
734         ret = -ENOBUFS;
735         spin_lock(&cookie->lock);
736
737         if (hlist_empty(&cookie->backing_objects))
738                 goto nobufs;
739         object = hlist_entry(cookie->backing_objects.first,
740                              struct fscache_object, cookie_link);
741         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
742                 goto nobufs;
743
744         /* add the page to the pending-storage radix tree on the backing
745          * object */
746         spin_lock(&object->lock);
747         spin_lock(&cookie->stores_lock);
748
749         _debug("store limit %llx", (unsigned long long) object->store_limit);
750
751         ret = radix_tree_insert(&cookie->stores, page->index, page);
752         if (ret < 0) {
753                 if (ret == -EEXIST)
754                         goto already_queued;
755                 _debug("insert failed %d", ret);
756                 goto nobufs_unlock_obj;
757         }
758
759         radix_tree_tag_set(&cookie->stores, page->index,
760                            FSCACHE_COOKIE_PENDING_TAG);
761         page_cache_get(page);
762
763         /* we only want one writer at a time, but we do need to queue new
764          * writers after exclusive ops */
765         if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
766                 goto already_pending;
767
768         spin_unlock(&cookie->stores_lock);
769         spin_unlock(&object->lock);
770
771         op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
772         op->store_limit = object->store_limit;
773
774         if (fscache_submit_op(object, &op->op) < 0)
775                 goto submit_failed;
776
777         spin_unlock(&cookie->lock);
778         radix_tree_preload_end();
779         fscache_stat(&fscache_n_store_ops);
780         fscache_stat(&fscache_n_stores_ok);
781
782         /* the slow work queue now carries its own ref on the object */
783         fscache_put_operation(&op->op);
784         _leave(" = 0");
785         return 0;
786
787 already_queued:
788         fscache_stat(&fscache_n_stores_again);
789 already_pending:
790         spin_unlock(&cookie->stores_lock);
791         spin_unlock(&object->lock);
792         spin_unlock(&cookie->lock);
793         radix_tree_preload_end();
794         kfree(op);
795         fscache_stat(&fscache_n_stores_ok);
796         _leave(" = 0");
797         return 0;
798
799 submit_failed:
800         spin_lock(&cookie->stores_lock);
801         radix_tree_delete(&cookie->stores, page->index);
802         spin_unlock(&cookie->stores_lock);
803         page_cache_release(page);
804         ret = -ENOBUFS;
805         goto nobufs;
806
807 nobufs_unlock_obj:
808         spin_unlock(&object->lock);
809 nobufs:
810         spin_unlock(&cookie->lock);
811         radix_tree_preload_end();
812         kfree(op);
813         fscache_stat(&fscache_n_stores_nobufs);
814         _leave(" = -ENOBUFS");
815         return -ENOBUFS;
816
817 nomem_free:
818         kfree(op);
819 nomem:
820         fscache_stat(&fscache_n_stores_oom);
821         _leave(" = -ENOMEM");
822         return -ENOMEM;
823 }
824 EXPORT_SYMBOL(__fscache_write_page);
825
826 /*
827  * remove a page from the cache
828  */
829 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
830 {
831         struct fscache_object *object;
832
833         _enter(",%p", page);
834
835         ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
836         ASSERTCMP(page, !=, NULL);
837
838         fscache_stat(&fscache_n_uncaches);
839
840         /* cache withdrawal may beat us to it */
841         if (!PageFsCache(page))
842                 goto done;
843
844         /* get the object */
845         spin_lock(&cookie->lock);
846
847         if (hlist_empty(&cookie->backing_objects)) {
848                 ClearPageFsCache(page);
849                 goto done_unlock;
850         }
851
852         object = hlist_entry(cookie->backing_objects.first,
853                              struct fscache_object, cookie_link);
854
855         /* there might now be stuff on disk we could read */
856         clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
857
858         /* only invoke the cache backend if we managed to mark the page
859          * uncached here; this deals with synchronisation vs withdrawal */
860         if (TestClearPageFsCache(page) &&
861             object->cache->ops->uncache_page) {
862                 /* the cache backend releases the cookie lock */
863                 fscache_stat(&fscache_n_cop_uncache_page);
864                 object->cache->ops->uncache_page(object, page);
865                 fscache_stat_d(&fscache_n_cop_uncache_page);
866                 goto done;
867         }
868
869 done_unlock:
870         spin_unlock(&cookie->lock);
871 done:
872         _leave("");
873 }
874 EXPORT_SYMBOL(__fscache_uncache_page);
875
876 /**
877  * fscache_mark_pages_cached - Mark pages as being cached
878  * @op: The retrieval op pages are being marked for
879  * @pagevec: The pages to be marked
880  *
881  * Mark a bunch of netfs pages as being cached.  After this is called,
882  * the netfs must call fscache_uncache_page() to remove the mark.
883  */
884 void fscache_mark_pages_cached(struct fscache_retrieval *op,
885                                struct pagevec *pagevec)
886 {
887         struct fscache_cookie *cookie = op->op.object->cookie;
888         unsigned long loop;
889
890 #ifdef CONFIG_FSCACHE_STATS
891         atomic_add(pagevec->nr, &fscache_n_marks);
892 #endif
893
894         for (loop = 0; loop < pagevec->nr; loop++) {
895                 struct page *page = pagevec->pages[loop];
896
897                 _debug("- mark %p{%lx}", page, page->index);
898                 if (TestSetPageFsCache(page)) {
899                         static bool once_only;
900                         if (!once_only) {
901                                 once_only = true;
902                                 printk(KERN_WARNING "FS-Cache:"
903                                        " Cookie type %s marked page %lx"
904                                        " multiple times\n",
905                                        cookie->def->name, page->index);
906                         }
907                 }
908         }
909
910         if (cookie->def->mark_pages_cached)
911                 cookie->def->mark_pages_cached(cookie->netfs_data,
912                                                op->mapping, pagevec);
913         pagevec_reinit(pagevec);
914 }
915 EXPORT_SYMBOL(fscache_mark_pages_cached);