linux/fs/dlm/lock.c
<<
>>
Prefs
   1/******************************************************************************
   2*******************************************************************************
   3**
   4**  Copyright (C) 2005-2010 Red Hat, Inc.  All rights reserved.
   5**
   6**  This copyrighted material is made available to anyone wishing to use,
   7**  modify, copy, or redistribute it subject to the terms and conditions
   8**  of the GNU General Public License v.2.
   9**
  10*******************************************************************************
  11******************************************************************************/
  12
  13/* Central locking logic has four stages:
  14
  15   dlm_lock()
  16   dlm_unlock()
  17
  18   request_lock(ls, lkb)
  19   convert_lock(ls, lkb)
  20   unlock_lock(ls, lkb)
  21   cancel_lock(ls, lkb)
  22
  23   _request_lock(r, lkb)
  24   _convert_lock(r, lkb)
  25   _unlock_lock(r, lkb)
  26   _cancel_lock(r, lkb)
  27
  28   do_request(r, lkb)
  29   do_convert(r, lkb)
  30   do_unlock(r, lkb)
  31   do_cancel(r, lkb)
  32
  33   Stage 1 (lock, unlock) is mainly about checking input args and
  34   splitting into one of the four main operations:
  35
  36       dlm_lock          = request_lock
  37       dlm_lock+CONVERT  = convert_lock
  38       dlm_unlock        = unlock_lock
  39       dlm_unlock+CANCEL = cancel_lock
  40
  41   Stage 2, xxxx_lock(), just finds and locks the relevant rsb which is
  42   provided to the next stage.
  43
  44   Stage 3, _xxxx_lock(), determines if the operation is local or remote.
  45   When remote, it calls send_xxxx(), when local it calls do_xxxx().
  46
  47   Stage 4, do_xxxx(), is the guts of the operation.  It manipulates the
  48   given rsb and lkb and queues callbacks.
  49
  50   For remote operations, send_xxxx() results in the corresponding do_xxxx()
  51   function being executed on the remote node.  The connecting send/receive
  52   calls on local (L) and remote (R) nodes:
  53
  54   L: send_xxxx()              ->  R: receive_xxxx()
  55                                   R: do_xxxx()
  56   L: receive_xxxx_reply()     <-  R: send_xxxx_reply()
  57*/
  58#include <linux/types.h>
  59#include <linux/rbtree.h>
  60#include <linux/slab.h>
  61#include "dlm_internal.h"
  62#include <linux/dlm_device.h>
  63#include "memory.h"
  64#include "lowcomms.h"
  65#include "requestqueue.h"
  66#include "util.h"
  67#include "dir.h"
  68#include "member.h"
  69#include "lockspace.h"
  70#include "ast.h"
  71#include "lock.h"
  72#include "rcom.h"
  73#include "recover.h"
  74#include "lvb_table.h"
  75#include "user.h"
  76#include "config.h"
  77
  78static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb);
  79static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb);
  80static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb);
  81static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb);
  82static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb);
  83static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode);
  84static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb);
  85static int send_remove(struct dlm_rsb *r);
  86static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb);
  87static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb);
  88static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
  89                                    struct dlm_message *ms);
  90static int receive_extralen(struct dlm_message *ms);
  91static void do_purge(struct dlm_ls *ls, int nodeid, int pid);
  92static void del_timeout(struct dlm_lkb *lkb);
  93static void toss_rsb(struct kref *kref);
  94
  95/*
  96 * Lock compatibilty matrix - thanks Steve
  97 * UN = Unlocked state. Not really a state, used as a flag
  98 * PD = Padding. Used to make the matrix a nice power of two in size
  99 * Other states are the same as the VMS DLM.
 100 * Usage: matrix[grmode+1][rqmode+1]  (although m[rq+1][gr+1] is the same)
 101 */
 102
 103static const int __dlm_compat_matrix[8][8] = {
 104      /* UN NL CR CW PR PW EX PD */
 105        {1, 1, 1, 1, 1, 1, 1, 0},       /* UN */
 106        {1, 1, 1, 1, 1, 1, 1, 0},       /* NL */
 107        {1, 1, 1, 1, 1, 1, 0, 0},       /* CR */
 108        {1, 1, 1, 1, 0, 0, 0, 0},       /* CW */
 109        {1, 1, 1, 0, 1, 0, 0, 0},       /* PR */
 110        {1, 1, 1, 0, 0, 0, 0, 0},       /* PW */
 111        {1, 1, 0, 0, 0, 0, 0, 0},       /* EX */
 112        {0, 0, 0, 0, 0, 0, 0, 0}        /* PD */
 113};
 114
 115/*
 116 * This defines the direction of transfer of LVB data.
 117 * Granted mode is the row; requested mode is the column.
 118 * Usage: matrix[grmode+1][rqmode+1]
 119 * 1 = LVB is returned to the caller
 120 * 0 = LVB is written to the resource
 121 * -1 = nothing happens to the LVB
 122 */
 123
 124const int dlm_lvb_operations[8][8] = {
 125        /* UN   NL  CR  CW  PR  PW  EX  PD*/
 126        {  -1,  1,  1,  1,  1,  1,  1, -1 }, /* UN */
 127        {  -1,  1,  1,  1,  1,  1,  1,  0 }, /* NL */
 128        {  -1, -1,  1,  1,  1,  1,  1,  0 }, /* CR */
 129        {  -1, -1, -1,  1,  1,  1,  1,  0 }, /* CW */
 130        {  -1, -1, -1, -1,  1,  1,  1,  0 }, /* PR */
 131        {  -1,  0,  0,  0,  0,  0,  1,  0 }, /* PW */
 132        {  -1,  0,  0,  0,  0,  0,  0,  0 }, /* EX */
 133        {  -1,  0,  0,  0,  0,  0,  0,  0 }  /* PD */
 134};
 135
 136#define modes_compat(gr, rq) \
 137        __dlm_compat_matrix[(gr)->lkb_grmode + 1][(rq)->lkb_rqmode + 1]
 138
 139int dlm_modes_compat(int mode1, int mode2)
 140{
 141        return __dlm_compat_matrix[mode1 + 1][mode2 + 1];
 142}
 143
 144/*
 145 * Compatibility matrix for conversions with QUECVT set.
 146 * Granted mode is the row; requested mode is the column.
 147 * Usage: matrix[grmode+1][rqmode+1]
 148 */
 149
 150static const int __quecvt_compat_matrix[8][8] = {
 151      /* UN NL CR CW PR PW EX PD */
 152        {0, 0, 0, 0, 0, 0, 0, 0},       /* UN */
 153        {0, 0, 1, 1, 1, 1, 1, 0},       /* NL */
 154        {0, 0, 0, 1, 1, 1, 1, 0},       /* CR */
 155        {0, 0, 0, 0, 1, 1, 1, 0},       /* CW */
 156        {0, 0, 0, 1, 0, 1, 1, 0},       /* PR */
 157        {0, 0, 0, 0, 0, 0, 1, 0},       /* PW */
 158        {0, 0, 0, 0, 0, 0, 0, 0},       /* EX */
 159        {0, 0, 0, 0, 0, 0, 0, 0}        /* PD */
 160};
 161
 162void dlm_print_lkb(struct dlm_lkb *lkb)
 163{
 164        printk(KERN_ERR "lkb: nodeid %d id %x remid %x exflags %x flags %x "
 165               "sts %d rq %d gr %d wait_type %d wait_nodeid %d seq %llu\n",
 166               lkb->lkb_nodeid, lkb->lkb_id, lkb->lkb_remid, lkb->lkb_exflags,
 167               lkb->lkb_flags, lkb->lkb_status, lkb->lkb_rqmode,
 168               lkb->lkb_grmode, lkb->lkb_wait_type, lkb->lkb_wait_nodeid,
 169               (unsigned long long)lkb->lkb_recover_seq);
 170}
 171
 172static void dlm_print_rsb(struct dlm_rsb *r)
 173{
 174        printk(KERN_ERR "rsb: nodeid %d master %d dir %d flags %lx first %x "
 175               "rlc %d name %s\n",
 176               r->res_nodeid, r->res_master_nodeid, r->res_dir_nodeid,
 177               r->res_flags, r->res_first_lkid, r->res_recover_locks_count,
 178               r->res_name);
 179}
 180
 181void dlm_dump_rsb(struct dlm_rsb *r)
 182{
 183        struct dlm_lkb *lkb;
 184
 185        dlm_print_rsb(r);
 186
 187        printk(KERN_ERR "rsb: root_list empty %d recover_list empty %d\n",
 188               list_empty(&r->res_root_list), list_empty(&r->res_recover_list));
 189        printk(KERN_ERR "rsb lookup list\n");
 190        list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup)
 191                dlm_print_lkb(lkb);
 192        printk(KERN_ERR "rsb grant queue:\n");
 193        list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue)
 194                dlm_print_lkb(lkb);
 195        printk(KERN_ERR "rsb convert queue:\n");
 196        list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue)
 197                dlm_print_lkb(lkb);
 198        printk(KERN_ERR "rsb wait queue:\n");
 199        list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue)
 200                dlm_print_lkb(lkb);
 201}
 202
 203/* Threads cannot use the lockspace while it's being recovered */
 204
 205static inline void dlm_lock_recovery(struct dlm_ls *ls)
 206{
 207        down_read(&ls->ls_in_recovery);
 208}
 209
 210void dlm_unlock_recovery(struct dlm_ls *ls)
 211{
 212        up_read(&ls->ls_in_recovery);
 213}
 214
 215int dlm_lock_recovery_try(struct dlm_ls *ls)
 216{
 217        return down_read_trylock(&ls->ls_in_recovery);
 218}
 219
 220static inline int can_be_queued(struct dlm_lkb *lkb)
 221{
 222        return !(lkb->lkb_exflags & DLM_LKF_NOQUEUE);
 223}
 224
 225static inline int force_blocking_asts(struct dlm_lkb *lkb)
 226{
 227        return (lkb->lkb_exflags & DLM_LKF_NOQUEUEBAST);
 228}
 229
 230static inline int is_demoted(struct dlm_lkb *lkb)
 231{
 232        return (lkb->lkb_sbflags & DLM_SBF_DEMOTED);
 233}
 234
 235static inline int is_altmode(struct dlm_lkb *lkb)
 236{
 237        return (lkb->lkb_sbflags & DLM_SBF_ALTMODE);
 238}
 239
 240static inline int is_granted(struct dlm_lkb *lkb)
 241{
 242        return (lkb->lkb_status == DLM_LKSTS_GRANTED);
 243}
 244
 245static inline int is_remote(struct dlm_rsb *r)
 246{
 247        DLM_ASSERT(r->res_nodeid >= 0, dlm_print_rsb(r););
 248        return !!r->res_nodeid;
 249}
 250
 251static inline int is_process_copy(struct dlm_lkb *lkb)
 252{
 253        return (lkb->lkb_nodeid && !(lkb->lkb_flags & DLM_IFL_MSTCPY));
 254}
 255
 256static inline int is_master_copy(struct dlm_lkb *lkb)
 257{
 258        return (lkb->lkb_flags & DLM_IFL_MSTCPY) ? 1 : 0;
 259}
 260
 261static inline int middle_conversion(struct dlm_lkb *lkb)
 262{
 263        if ((lkb->lkb_grmode==DLM_LOCK_PR && lkb->lkb_rqmode==DLM_LOCK_CW) ||
 264            (lkb->lkb_rqmode==DLM_LOCK_PR && lkb->lkb_grmode==DLM_LOCK_CW))
 265                return 1;
 266        return 0;
 267}
 268
 269static inline int down_conversion(struct dlm_lkb *lkb)
 270{
 271        return (!middle_conversion(lkb) && lkb->lkb_rqmode < lkb->lkb_grmode);
 272}
 273
 274static inline int is_overlap_unlock(struct dlm_lkb *lkb)
 275{
 276        return lkb->lkb_flags & DLM_IFL_OVERLAP_UNLOCK;
 277}
 278
 279static inline int is_overlap_cancel(struct dlm_lkb *lkb)
 280{
 281        return lkb->lkb_flags & DLM_IFL_OVERLAP_CANCEL;
 282}
 283
 284static inline int is_overlap(struct dlm_lkb *lkb)
 285{
 286        return (lkb->lkb_flags & (DLM_IFL_OVERLAP_UNLOCK |
 287                                  DLM_IFL_OVERLAP_CANCEL));
 288}
 289
 290static void queue_cast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
 291{
 292        if (is_master_copy(lkb))
 293                return;
 294
 295        del_timeout(lkb);
 296
 297        DLM_ASSERT(lkb->lkb_lksb, dlm_print_lkb(lkb););
 298
 299        /* if the operation was a cancel, then return -DLM_ECANCEL, if a
 300           timeout caused the cancel then return -ETIMEDOUT */
 301        if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_TIMEOUT_CANCEL)) {
 302                lkb->lkb_flags &= ~DLM_IFL_TIMEOUT_CANCEL;
 303                rv = -ETIMEDOUT;
 304        }
 305
 306        if (rv == -DLM_ECANCEL && (lkb->lkb_flags & DLM_IFL_DEADLOCK_CANCEL)) {
 307                lkb->lkb_flags &= ~DLM_IFL_DEADLOCK_CANCEL;
 308                rv = -EDEADLK;
 309        }
 310
 311        dlm_add_cb(lkb, DLM_CB_CAST, lkb->lkb_grmode, rv, lkb->lkb_sbflags);
 312}
 313
 314static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb)
 315{
 316        queue_cast(r, lkb,
 317                   is_overlap_unlock(lkb) ? -DLM_EUNLOCK : -DLM_ECANCEL);
 318}
 319
 320static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode)
 321{
 322        if (is_master_copy(lkb)) {
 323                send_bast(r, lkb, rqmode);
 324        } else {
 325                dlm_add_cb(lkb, DLM_CB_BAST, rqmode, 0, 0);
 326        }
 327}
 328
 329/*
 330 * Basic operations on rsb's and lkb's
 331 */
 332
 333/* This is only called to add a reference when the code already holds
 334   a valid reference to the rsb, so there's no need for locking. */
 335
 336static inline void hold_rsb(struct dlm_rsb *r)
 337{
 338        kref_get(&r->res_ref);
 339}
 340
 341void dlm_hold_rsb(struct dlm_rsb *r)
 342{
 343        hold_rsb(r);
 344}
 345
 346/* When all references to the rsb are gone it's transferred to
 347   the tossed list for later disposal. */
 348
 349static void put_rsb(struct dlm_rsb *r)
 350{
 351        struct dlm_ls *ls = r->res_ls;
 352        uint32_t bucket = r->res_bucket;
 353
 354        spin_lock(&ls->ls_rsbtbl[bucket].lock);
 355        kref_put(&r->res_ref, toss_rsb);
 356        spin_unlock(&ls->ls_rsbtbl[bucket].lock);
 357}
 358
 359void dlm_put_rsb(struct dlm_rsb *r)
 360{
 361        put_rsb(r);
 362}
 363
 364static int pre_rsb_struct(struct dlm_ls *ls)
 365{
 366        struct dlm_rsb *r1, *r2;
 367        int count = 0;
 368
 369        spin_lock(&ls->ls_new_rsb_spin);
 370        if (ls->ls_new_rsb_count > dlm_config.ci_new_rsb_count / 2) {
 371                spin_unlock(&ls->ls_new_rsb_spin);
 372                return 0;
 373        }
 374        spin_unlock(&ls->ls_new_rsb_spin);
 375
 376        r1 = dlm_allocate_rsb(ls);
 377        r2 = dlm_allocate_rsb(ls);
 378
 379        spin_lock(&ls->ls_new_rsb_spin);
 380        if (r1) {
 381                list_add(&r1->res_hashchain, &ls->ls_new_rsb);
 382                ls->ls_new_rsb_count++;
 383        }
 384        if (r2) {
 385                list_add(&r2->res_hashchain, &ls->ls_new_rsb);
 386                ls->ls_new_rsb_count++;
 387        }
 388        count = ls->ls_new_rsb_count;
 389        spin_unlock(&ls->ls_new_rsb_spin);
 390
 391        if (!count)
 392                return -ENOMEM;
 393        return 0;
 394}
 395
 396/* If ls->ls_new_rsb is empty, return -EAGAIN, so the caller can
 397   unlock any spinlocks, go back and call pre_rsb_struct again.
 398   Otherwise, take an rsb off the list and return it. */
 399
 400static int get_rsb_struct(struct dlm_ls *ls, char *name, int len,
 401                          struct dlm_rsb **r_ret)
 402{
 403        struct dlm_rsb *r;
 404        int count;
 405
 406        spin_lock(&ls->ls_new_rsb_spin);
 407        if (list_empty(&ls->ls_new_rsb)) {
 408                count = ls->ls_new_rsb_count;
 409                spin_unlock(&ls->ls_new_rsb_spin);
 410                log_debug(ls, "find_rsb retry %d %d %s",
 411                          count, dlm_config.ci_new_rsb_count, name);
 412                return -EAGAIN;
 413        }
 414
 415        r = list_first_entry(&ls->ls_new_rsb, struct dlm_rsb, res_hashchain);
 416        list_del(&r->res_hashchain);
 417        /* Convert the empty list_head to a NULL rb_node for tree usage: */
 418        memset(&r->res_hashnode, 0, sizeof(struct rb_node));
 419        ls->ls_new_rsb_count--;
 420        spin_unlock(&ls->ls_new_rsb_spin);
 421
 422        r->res_ls = ls;
 423        r->res_length = len;
 424        memcpy(r->res_name, name, len);
 425        mutex_init(&r->res_mutex);
 426
 427        INIT_LIST_HEAD(&r->res_lookup);
 428        INIT_LIST_HEAD(&r->res_grantqueue);
 429        INIT_LIST_HEAD(&r->res_convertqueue);
 430        INIT_LIST_HEAD(&r->res_waitqueue);
 431        INIT_LIST_HEAD(&r->res_root_list);
 432        INIT_LIST_HEAD(&r->res_recover_list);
 433
 434        *r_ret = r;
 435        return 0;
 436}
 437
 438static int rsb_cmp(struct dlm_rsb *r, const char *name, int nlen)
 439{
 440        char maxname[DLM_RESNAME_MAXLEN];
 441
 442        memset(maxname, 0, DLM_RESNAME_MAXLEN);
 443        memcpy(maxname, name, nlen);
 444        return memcmp(r->res_name, maxname, DLM_RESNAME_MAXLEN);
 445}
 446
 447int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len,
 448                        struct dlm_rsb **r_ret)
 449{
 450        struct rb_node *node = tree->rb_node;
 451        struct dlm_rsb *r;
 452        int rc;
 453
 454        while (node) {
 455                r = rb_entry(node, struct dlm_rsb, res_hashnode);
 456                rc = rsb_cmp(r, name, len);
 457                if (rc < 0)
 458                        node = node->rb_left;
 459                else if (rc > 0)
 460                        node = node->rb_right;
 461                else
 462                        goto found;
 463        }
 464        *r_ret = NULL;
 465        return -EBADR;
 466
 467 found:
 468        *r_ret = r;
 469        return 0;
 470}
 471
 472static int rsb_insert(struct dlm_rsb *rsb, struct rb_root *tree)
 473{
 474        struct rb_node **newn = &tree->rb_node;
 475        struct rb_node *parent = NULL;
 476        int rc;
 477
 478        while (*newn) {
 479                struct dlm_rsb *cur = rb_entry(*newn, struct dlm_rsb,
 480                                               res_hashnode);
 481
 482                parent = *newn;
 483                rc = rsb_cmp(cur, rsb->res_name, rsb->res_length);
 484                if (rc < 0)
 485                        newn = &parent->rb_left;
 486                else if (rc > 0)
 487                        newn = &parent->rb_right;
 488                else {
 489                        log_print("rsb_insert match");
 490                        dlm_dump_rsb(rsb);
 491                        dlm_dump_rsb(cur);
 492                        return -EEXIST;
 493                }
 494        }
 495
 496        rb_link_node(&rsb->res_hashnode, parent, newn);
 497        rb_insert_color(&rsb->res_hashnode, tree);
 498        return 0;
 499}
 500
 501/*
 502 * Find rsb in rsbtbl and potentially create/add one
 503 *
 504 * Delaying the release of rsb's has a similar benefit to applications keeping
 505 * NL locks on an rsb, but without the guarantee that the cached master value
 506 * will still be valid when the rsb is reused.  Apps aren't always smart enough
 507 * to keep NL locks on an rsb that they may lock again shortly; this can lead
 508 * to excessive master lookups and removals if we don't delay the release.
 509 *
 510 * Searching for an rsb means looking through both the normal list and toss
 511 * list.  When found on the toss list the rsb is moved to the normal list with
 512 * ref count of 1; when found on normal list the ref count is incremented.
 513 *
 514 * rsb's on the keep list are being used locally and refcounted.
 515 * rsb's on the toss list are not being used locally, and are not refcounted.
 516 *
 517 * The toss list rsb's were either
 518 * - previously used locally but not any more (were on keep list, then
 519 *   moved to toss list when last refcount dropped)
 520 * - created and put on toss list as a directory record for a lookup
 521 *   (we are the dir node for the res, but are not using the res right now,
 522 *   but some other node is)
 523 *
 524 * The purpose of find_rsb() is to return a refcounted rsb for local use.
 525 * So, if the given rsb is on the toss list, it is moved to the keep list
 526 * before being returned.
 527 *
 528 * toss_rsb() happens when all local usage of the rsb is done, i.e. no
 529 * more refcounts exist, so the rsb is moved from the keep list to the
 530 * toss list.
 531 *
 532 * rsb's on both keep and toss lists are used for doing a name to master
 533 * lookups.  rsb's that are in use locally (and being refcounted) are on
 534 * the keep list, rsb's that are not in use locally (not refcounted) and
 535 * only exist for name/master lookups are on the toss list.
 536 *
 537 * rsb's on the toss list who's dir_nodeid is not local can have stale
 538 * name/master mappings.  So, remote requests on such rsb's can potentially
 539 * return with an error, which means the mapping is stale and needs to
 540 * be updated with a new lookup.  (The idea behind MASTER UNCERTAIN and
 541 * first_lkid is to keep only a single outstanding request on an rsb
 542 * while that rsb has a potentially stale master.)
 543 */
 544
 545static int find_rsb_dir(struct dlm_ls *ls, char *name, int len,
 546                        uint32_t hash, uint32_t b,
 547                        int dir_nodeid, int from_nodeid,
 548                        unsigned int flags, struct dlm_rsb **r_ret)
 549{
 550        struct dlm_rsb *r = NULL;
 551        int our_nodeid = dlm_our_nodeid();
 552        int from_local = 0;
 553        int from_other = 0;
 554        int from_dir = 0;
 555        int create = 0;
 556        int error;
 557
 558        if (flags & R_RECEIVE_REQUEST) {
 559                if (from_nodeid == dir_nodeid)
 560                        from_dir = 1;
 561                else
 562                        from_other = 1;
 563        } else if (flags & R_REQUEST) {
 564                from_local = 1;
 565        }
 566
 567        /*
 568         * flags & R_RECEIVE_RECOVER is from dlm_recover_master_copy, so
 569         * from_nodeid has sent us a lock in dlm_recover_locks, believing
 570         * we're the new master.  Our local recovery may not have set
 571         * res_master_nodeid to our_nodeid yet, so allow either.  Don't
 572         * create the rsb; dlm_recover_process_copy() will handle EBADR
 573         * by resending.
 574         *
 575         * If someone sends us a request, we are the dir node, and we do
 576         * not find the rsb anywhere, then recreate it.  This happens if
 577         * someone sends us a request after we have removed/freed an rsb
 578         * from our toss list.  (They sent a request instead of lookup
 579         * because they are using an rsb from their toss list.)
 580         */
 581
 582        if (from_local || from_dir ||
 583            (from_other && (dir_nodeid == our_nodeid))) {
 584                create = 1;
 585        }
 586
 587 retry:
 588        if (create) {
 589                error = pre_rsb_struct(ls);
 590                if (error < 0)
 591                        goto out;
 592        }
 593
 594        spin_lock(&ls->ls_rsbtbl[b].lock);
 595
 596        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r);
 597        if (error)
 598                goto do_toss;
 599        
 600        /*
 601         * rsb is active, so we can't check master_nodeid without lock_rsb.
 602         */
 603
 604        kref_get(&r->res_ref);
 605        error = 0;
 606        goto out_unlock;
 607
 608
 609 do_toss:
 610        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r);
 611        if (error)
 612                goto do_new;
 613
 614        /*
 615         * rsb found inactive (master_nodeid may be out of date unless
 616         * we are the dir_nodeid or were the master)  No other thread
 617         * is using this rsb because it's on the toss list, so we can
 618         * look at or update res_master_nodeid without lock_rsb.
 619         */
 620
 621        if ((r->res_master_nodeid != our_nodeid) && from_other) {
 622                /* our rsb was not master, and another node (not the dir node)
 623                   has sent us a request */
 624                log_debug(ls, "find_rsb toss from_other %d master %d dir %d %s",
 625                          from_nodeid, r->res_master_nodeid, dir_nodeid,
 626                          r->res_name);
 627                error = -ENOTBLK;
 628                goto out_unlock;
 629        }
 630
 631        if ((r->res_master_nodeid != our_nodeid) && from_dir) {
 632                /* don't think this should ever happen */
 633                log_error(ls, "find_rsb toss from_dir %d master %d",
 634                          from_nodeid, r->res_master_nodeid);
 635                dlm_print_rsb(r);
 636                /* fix it and go on */
 637                r->res_master_nodeid = our_nodeid;
 638                r->res_nodeid = 0;
 639                rsb_clear_flag(r, RSB_MASTER_UNCERTAIN);
 640                r->res_first_lkid = 0;
 641        }
 642
 643        if (from_local && (r->res_master_nodeid != our_nodeid)) {
 644                /* Because we have held no locks on this rsb,
 645                   res_master_nodeid could have become stale. */
 646                rsb_set_flag(r, RSB_MASTER_UNCERTAIN);
 647                r->res_first_lkid = 0;
 648        }
 649
 650        rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss);
 651        error = rsb_insert(r, &ls->ls_rsbtbl[b].keep);
 652        goto out_unlock;
 653
 654
 655 do_new:
 656        /*
 657         * rsb not found
 658         */
 659
 660        if (error == -EBADR && !create)
 661                goto out_unlock;
 662
 663        error = get_rsb_struct(ls, name, len, &r);
 664        if (error == -EAGAIN) {
 665                spin_unlock(&ls->ls_rsbtbl[b].lock);
 666                goto retry;
 667        }
 668        if (error)
 669                goto out_unlock;
 670
 671        r->res_hash = hash;
 672        r->res_bucket = b;
 673        r->res_dir_nodeid = dir_nodeid;
 674        kref_init(&r->res_ref);
 675
 676        if (from_dir) {
 677                /* want to see how often this happens */
 678                log_debug(ls, "find_rsb new from_dir %d recreate %s",
 679                          from_nodeid, r->res_name);
 680                r->res_master_nodeid = our_nodeid;
 681                r->res_nodeid = 0;
 682                goto out_add;
 683        }
 684
 685        if (from_other && (dir_nodeid != our_nodeid)) {
 686                /* should never happen */
 687                log_error(ls, "find_rsb new from_other %d dir %d our %d %s",
 688                          from_nodeid, dir_nodeid, our_nodeid, r->res_name);
 689                dlm_free_rsb(r);
 690                error = -ENOTBLK;
 691                goto out_unlock;
 692        }
 693
 694        if (from_other) {
 695                log_debug(ls, "find_rsb new from_other %d dir %d %s",
 696                          from_nodeid, dir_nodeid, r->res_name);
 697        }
 698
 699        if (dir_nodeid == our_nodeid) {
 700                /* When we are the dir nodeid, we can set the master
 701                   node immediately */
 702                r->res_master_nodeid = our_nodeid;
 703                r->res_nodeid = 0;
 704        } else {
 705                /* set_master will send_lookup to dir_nodeid */
 706                r->res_master_nodeid = 0;
 707                r->res_nodeid = -1;
 708        }
 709
 710 out_add:
 711        error = rsb_insert(r, &ls->ls_rsbtbl[b].keep);
 712 out_unlock:
 713        spin_unlock(&ls->ls_rsbtbl[b].lock);
 714 out:
 715        *r_ret = r;
 716        return error;
 717}
 718
 719/* During recovery, other nodes can send us new MSTCPY locks (from
 720   dlm_recover_locks) before we've made ourself master (in
 721   dlm_recover_masters). */
 722
 723static int find_rsb_nodir(struct dlm_ls *ls, char *name, int len,
 724                          uint32_t hash, uint32_t b,
 725                          int dir_nodeid, int from_nodeid,
 726                          unsigned int flags, struct dlm_rsb **r_ret)
 727{
 728        struct dlm_rsb *r = NULL;
 729        int our_nodeid = dlm_our_nodeid();
 730        int recover = (flags & R_RECEIVE_RECOVER);
 731        int error;
 732
 733 retry:
 734        error = pre_rsb_struct(ls);
 735        if (error < 0)
 736                goto out;
 737
 738        spin_lock(&ls->ls_rsbtbl[b].lock);
 739
 740        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r);
 741        if (error)
 742                goto do_toss;
 743
 744        /*
 745         * rsb is active, so we can't check master_nodeid without lock_rsb.
 746         */
 747
 748        kref_get(&r->res_ref);
 749        goto out_unlock;
 750
 751
 752 do_toss:
 753        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r);
 754        if (error)
 755                goto do_new;
 756
 757        /*
 758         * rsb found inactive. No other thread is using this rsb because
 759         * it's on the toss list, so we can look at or update
 760         * res_master_nodeid without lock_rsb.
 761         */
 762
 763        if (!recover && (r->res_master_nodeid != our_nodeid) && from_nodeid) {
 764                /* our rsb is not master, and another node has sent us a
 765                   request; this should never happen */
 766                log_error(ls, "find_rsb toss from_nodeid %d master %d dir %d",
 767                          from_nodeid, r->res_master_nodeid, dir_nodeid);
 768                dlm_print_rsb(r);
 769                error = -ENOTBLK;
 770                goto out_unlock;
 771        }
 772
 773        if (!recover && (r->res_master_nodeid != our_nodeid) &&
 774            (dir_nodeid == our_nodeid)) {
 775                /* our rsb is not master, and we are dir; may as well fix it;
 776                   this should never happen */
 777                log_error(ls, "find_rsb toss our %d master %d dir %d",
 778                          our_nodeid, r->res_master_nodeid, dir_nodeid);
 779                dlm_print_rsb(r);
 780                r->res_master_nodeid = our_nodeid;
 781                r->res_nodeid = 0;
 782        }
 783
 784        rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss);
 785        error = rsb_insert(r, &ls->ls_rsbtbl[b].keep);
 786        goto out_unlock;
 787
 788
 789 do_new:
 790        /*
 791         * rsb not found
 792         */
 793
 794        error = get_rsb_struct(ls, name, len, &r);
 795        if (error == -EAGAIN) {
 796                spin_unlock(&ls->ls_rsbtbl[b].lock);
 797                goto retry;
 798        }
 799        if (error)
 800                goto out_unlock;
 801
 802        r->res_hash = hash;
 803        r->res_bucket = b;
 804        r->res_dir_nodeid = dir_nodeid;
 805        r->res_master_nodeid = dir_nodeid;
 806        r->res_nodeid = (dir_nodeid == our_nodeid) ? 0 : dir_nodeid;
 807        kref_init(&r->res_ref);
 808
 809        error = rsb_insert(r, &ls->ls_rsbtbl[b].keep);
 810 out_unlock:
 811        spin_unlock(&ls->ls_rsbtbl[b].lock);
 812 out:
 813        *r_ret = r;
 814        return error;
 815}
 816
 817static int find_rsb(struct dlm_ls *ls, char *name, int len, int from_nodeid,
 818                    unsigned int flags, struct dlm_rsb **r_ret)
 819{
 820        uint32_t hash, b;
 821        int dir_nodeid;
 822
 823        if (len > DLM_RESNAME_MAXLEN)
 824                return -EINVAL;
 825
 826        hash = jhash(name, len, 0);
 827        b = hash & (ls->ls_rsbtbl_size - 1);
 828
 829        dir_nodeid = dlm_hash2nodeid(ls, hash);
 830
 831        if (dlm_no_directory(ls))
 832                return find_rsb_nodir(ls, name, len, hash, b, dir_nodeid,
 833                                      from_nodeid, flags, r_ret);
 834        else
 835                return find_rsb_dir(ls, name, len, hash, b, dir_nodeid,
 836                                      from_nodeid, flags, r_ret);
 837}
 838
 839/* we have received a request and found that res_master_nodeid != our_nodeid,
 840   so we need to return an error or make ourself the master */
 841
 842static int validate_master_nodeid(struct dlm_ls *ls, struct dlm_rsb *r,
 843                                  int from_nodeid)
 844{
 845        if (dlm_no_directory(ls)) {
 846                log_error(ls, "find_rsb keep from_nodeid %d master %d dir %d",
 847                          from_nodeid, r->res_master_nodeid,
 848                          r->res_dir_nodeid);
 849                dlm_print_rsb(r);
 850                return -ENOTBLK;
 851        }
 852
 853        if (from_nodeid != r->res_dir_nodeid) {
 854                /* our rsb is not master, and another node (not the dir node)
 855                   has sent us a request.  this is much more common when our
 856                   master_nodeid is zero, so limit debug to non-zero.  */
 857
 858                if (r->res_master_nodeid) {
 859                        log_debug(ls, "validate master from_other %d master %d "
 860                                  "dir %d first %x %s", from_nodeid,
 861                                  r->res_master_nodeid, r->res_dir_nodeid,
 862                                  r->res_first_lkid, r->res_name);
 863                }
 864                return -ENOTBLK;
 865        } else {
 866                /* our rsb is not master, but the dir nodeid has sent us a
 867                   request; this could happen with master 0 / res_nodeid -1 */
 868
 869                if (r->res_master_nodeid) {
 870                        log_error(ls, "validate master from_dir %d master %d "
 871                                  "first %x %s",
 872                                  from_nodeid, r->res_master_nodeid,
 873                                  r->res_first_lkid, r->res_name);
 874                }
 875
 876                r->res_master_nodeid = dlm_our_nodeid();
 877                r->res_nodeid = 0;
 878                return 0;
 879        }
 880}
 881
 882/*
 883 * We're the dir node for this res and another node wants to know the
 884 * master nodeid.  During normal operation (non recovery) this is only
 885 * called from receive_lookup(); master lookups when the local node is
 886 * the dir node are done by find_rsb().
 887 *
 888 * normal operation, we are the dir node for a resource
 889 * . _request_lock
 890 * . set_master
 891 * . send_lookup
 892 * . receive_lookup
 893 * . dlm_master_lookup flags 0
 894 *
 895 * recover directory, we are rebuilding dir for all resources
 896 * . dlm_recover_directory
 897 * . dlm_rcom_names
 898 *   remote node sends back the rsb names it is master of and we are dir of
 899 * . dlm_master_lookup RECOVER_DIR (fix_master 0, from_master 1)
 900 *   we either create new rsb setting remote node as master, or find existing
 901 *   rsb and set master to be the remote node.
 902 *
 903 * recover masters, we are finding the new master for resources
 904 * . dlm_recover_masters
 905 * . recover_master
 906 * . dlm_send_rcom_lookup
 907 * . receive_rcom_lookup
 908 * . dlm_master_lookup RECOVER_MASTER (fix_master 1, from_master 0)
 909 */
 910
 911int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, char *name, int len,
 912                      unsigned int flags, int *r_nodeid, int *result)
 913{
 914        struct dlm_rsb *r = NULL;
 915        uint32_t hash, b;
 916        int from_master = (flags & DLM_LU_RECOVER_DIR);
 917        int fix_master = (flags & DLM_LU_RECOVER_MASTER);
 918        int our_nodeid = dlm_our_nodeid();
 919        int dir_nodeid, error, toss_list = 0;
 920
 921        if (len > DLM_RESNAME_MAXLEN)
 922                return -EINVAL;
 923
 924        if (from_nodeid == our_nodeid) {
 925                log_error(ls, "dlm_master_lookup from our_nodeid %d flags %x",
 926                          our_nodeid, flags);
 927                return -EINVAL;
 928        }
 929
 930        hash = jhash(name, len, 0);
 931        b = hash & (ls->ls_rsbtbl_size - 1);
 932
 933        dir_nodeid = dlm_hash2nodeid(ls, hash);
 934        if (dir_nodeid != our_nodeid) {
 935                log_error(ls, "dlm_master_lookup from %d dir %d our %d h %x %d",
 936                          from_nodeid, dir_nodeid, our_nodeid, hash,
 937                          ls->ls_num_nodes);
 938                *r_nodeid = -1;
 939                return -EINVAL;
 940        }
 941
 942 retry:
 943        error = pre_rsb_struct(ls);
 944        if (error < 0)
 945                return error;
 946
 947        spin_lock(&ls->ls_rsbtbl[b].lock);
 948        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r);
 949        if (!error) {
 950                /* because the rsb is active, we need to lock_rsb before
 951                   checking/changing re_master_nodeid */
 952
 953                hold_rsb(r);
 954                spin_unlock(&ls->ls_rsbtbl[b].lock);
 955                lock_rsb(r);
 956                goto found;
 957        }
 958
 959        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r);
 960        if (error)
 961                goto not_found;
 962
 963        /* because the rsb is inactive (on toss list), it's not refcounted
 964           and lock_rsb is not used, but is protected by the rsbtbl lock */
 965
 966        toss_list = 1;
 967 found:
 968        if (r->res_dir_nodeid != our_nodeid) {
 969                /* should not happen, but may as well fix it and carry on */
 970                log_error(ls, "dlm_master_lookup res_dir %d our %d %s",
 971                          r->res_dir_nodeid, our_nodeid, r->res_name);
 972                r->res_dir_nodeid = our_nodeid;
 973        }
 974
 975        if (fix_master && dlm_is_removed(ls, r->res_master_nodeid)) {
 976                /* Recovery uses this function to set a new master when
 977                   the previous master failed.  Setting NEW_MASTER will
 978                   force dlm_recover_masters to call recover_master on this
 979                   rsb even though the res_nodeid is no longer removed. */
 980
 981                r->res_master_nodeid = from_nodeid;
 982                r->res_nodeid = from_nodeid;
 983                rsb_set_flag(r, RSB_NEW_MASTER);
 984
 985                if (toss_list) {
 986                        /* I don't think we should ever find it on toss list. */
 987                        log_error(ls, "dlm_master_lookup fix_master on toss");
 988                        dlm_dump_rsb(r);
 989                }
 990        }
 991
 992        if (from_master && (r->res_master_nodeid != from_nodeid)) {
 993                /* this will happen if from_nodeid became master during
 994                   a previous recovery cycle, and we aborted the previous
 995                   cycle before recovering this master value */
 996
 997                log_limit(ls, "dlm_master_lookup from_master %d "
 998                          "master_nodeid %d res_nodeid %d first %x %s",
 999                          from_nodeid, r->res_master_nodeid, r->res_nodeid,
1000                          r->res_first_lkid, r->res_name);
1001
1002                if (r->res_master_nodeid == our_nodeid) {
1003                        log_error(ls, "from_master %d our_master", from_nodeid);
1004                        dlm_dump_rsb(r);
1005                        dlm_send_rcom_lookup_dump(r, from_nodeid);
1006                        goto out_found;
1007                }
1008
1009                r->res_master_nodeid = from_nodeid;
1010                r->res_nodeid = from_nodeid;
1011                rsb_set_flag(r, RSB_NEW_MASTER);
1012        }
1013
1014        if (!r->res_master_nodeid) {
1015                /* this will happen if recovery happens while we're looking
1016                   up the master for this rsb */
1017
1018                log_debug(ls, "dlm_master_lookup master 0 to %d first %x %s",
1019                          from_nodeid, r->res_first_lkid, r->res_name);
1020                r->res_master_nodeid = from_nodeid;
1021                r->res_nodeid = from_nodeid;
1022        }
1023
1024        if (!from_master && !fix_master &&
1025            (r->res_master_nodeid == from_nodeid)) {
1026                /* this can happen when the master sends remove, the dir node
1027                   finds the rsb on the keep list and ignores the remove,
1028                   and the former master sends a lookup */
1029
1030                log_limit(ls, "dlm_master_lookup from master %d flags %x "
1031                          "first %x %s", from_nodeid, flags,
1032                          r->res_first_lkid, r->res_name);
1033        }
1034
1035 out_found:
1036        *r_nodeid = r->res_master_nodeid;
1037        if (result)
1038                *result = DLM_LU_MATCH;
1039
1040        if (toss_list) {
1041                r->res_toss_time = jiffies;
1042                /* the rsb was inactive (on toss list) */
1043                spin_unlock(&ls->ls_rsbtbl[b].lock);
1044        } else {
1045                /* the rsb was active */
1046                unlock_rsb(r);
1047                put_rsb(r);
1048        }
1049        return 0;
1050
1051 not_found:
1052        error = get_rsb_struct(ls, name, len, &r);
1053        if (error == -EAGAIN) {
1054                spin_unlock(&ls->ls_rsbtbl[b].lock);
1055                goto retry;
1056        }
1057        if (error)
1058                goto out_unlock;
1059
1060        r->res_hash = hash;
1061        r->res_bucket = b;
1062        r->res_dir_nodeid = our_nodeid;
1063        r->res_master_nodeid = from_nodeid;
1064        r->res_nodeid = from_nodeid;
1065        kref_init(&r->res_ref);
1066        r->res_toss_time = jiffies;
1067
1068        error = rsb_insert(r, &ls->ls_rsbtbl[b].toss);
1069        if (error) {
1070                /* should never happen */
1071                dlm_free_rsb(r);
1072                spin_unlock(&ls->ls_rsbtbl[b].lock);
1073                goto retry;
1074        }
1075
1076        if (result)
1077                *result = DLM_LU_ADD;
1078        *r_nodeid = from_nodeid;
1079        error = 0;
1080 out_unlock:
1081        spin_unlock(&ls->ls_rsbtbl[b].lock);
1082        return error;
1083}
1084
1085static void dlm_dump_rsb_hash(struct dlm_ls *ls, uint32_t hash)
1086{
1087        struct rb_node *n;
1088        struct dlm_rsb *r;
1089        int i;
1090
1091        for (i = 0; i < ls->ls_rsbtbl_size; i++) {
1092                spin_lock(&ls->ls_rsbtbl[i].lock);
1093                for (n = rb_first(&ls->ls_rsbtbl[i].keep); n; n = rb_next(n)) {
1094                        r = rb_entry(n, struct dlm_rsb, res_hashnode);
1095                        if (r->res_hash == hash)
1096                                dlm_dump_rsb(r);
1097                }
1098                spin_unlock(&ls->ls_rsbtbl[i].lock);
1099        }
1100}
1101
1102void dlm_dump_rsb_name(struct dlm_ls *ls, char *name, int len)
1103{
1104        struct dlm_rsb *r = NULL;
1105        uint32_t hash, b;
1106        int error;
1107
1108        hash = jhash(name, len, 0);
1109        b = hash & (ls->ls_rsbtbl_size - 1);
1110
1111        spin_lock(&ls->ls_rsbtbl[b].lock);
1112        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r);
1113        if (!error)
1114                goto out_dump;
1115
1116        error = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r);
1117        if (error)
1118                goto out;
1119 out_dump:
1120        dlm_dump_rsb(r);
1121 out:
1122        spin_unlock(&ls->ls_rsbtbl[b].lock);
1123}
1124
1125static void toss_rsb(struct kref *kref)
1126{
1127        struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref);
1128        struct dlm_ls *ls = r->res_ls;
1129
1130        DLM_ASSERT(list_empty(&r->res_root_list), dlm_print_rsb(r););
1131        kref_init(&r->res_ref);
1132        rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[r->res_bucket].keep);
1133        rsb_insert(r, &ls->ls_rsbtbl[r->res_bucket].toss);
1134        r->res_toss_time = jiffies;
1135        ls->ls_rsbtbl[r->res_bucket].flags |= DLM_RTF_SHRINK;
1136        if (r->res_lvbptr) {
1137                dlm_free_lvb(r->res_lvbptr);
1138                r->res_lvbptr = NULL;
1139        }
1140}
1141
1142/* See comment for unhold_lkb */
1143
1144static void unhold_rsb(struct dlm_rsb *r)
1145{
1146        int rv;
1147        rv = kref_put(&r->res_ref, toss_rsb);
1148        DLM_ASSERT(!rv, dlm_dump_rsb(r););
1149}
1150
1151static void kill_rsb(struct kref *kref)
1152{
1153        struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref);
1154
1155        /* All work is done after the return from kref_put() so we
1156           can release the write_lock before the remove and free. */
1157
1158        DLM_ASSERT(list_empty(&r->res_lookup), dlm_dump_rsb(r););
1159        DLM_ASSERT(list_empty(&r->res_grantqueue), dlm_dump_rsb(r););
1160        DLM_ASSERT(list_empty(&r->res_convertqueue), dlm_dump_rsb(r););
1161        DLM_ASSERT(list_empty(&r->res_waitqueue), dlm_dump_rsb(r););
1162        DLM_ASSERT(list_empty(&r->res_root_list), dlm_dump_rsb(r););
1163        DLM_ASSERT(list_empty(&r->res_recover_list), dlm_dump_rsb(r););
1164}
1165
1166/* Attaching/detaching lkb's from rsb's is for rsb reference counting.
1167   The rsb must exist as long as any lkb's for it do. */
1168
1169static void attach_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
1170{
1171        hold_rsb(r);
1172        lkb->lkb_resource = r;
1173}
1174
1175static void detach_lkb(struct dlm_lkb *lkb)
1176{
1177        if (lkb->lkb_resource) {
1178                put_rsb(lkb->lkb_resource);
1179                lkb->lkb_resource = NULL;
1180        }
1181}
1182
1183static int create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret)
1184{
1185        struct dlm_lkb *lkb;
1186        int rv;
1187
1188        lkb = dlm_allocate_lkb(ls);
1189        if (!lkb)
1190                return -ENOMEM;
1191
1192        lkb->lkb_nodeid = -1;
1193        lkb->lkb_grmode = DLM_LOCK_IV;
1194        kref_init(&lkb->lkb_ref);
1195        INIT_LIST_HEAD(&lkb->lkb_ownqueue);
1196        INIT_LIST_HEAD(&lkb->lkb_rsb_lookup);
1197        INIT_LIST_HEAD(&lkb->lkb_time_list);
1198        INIT_LIST_HEAD(&lkb->lkb_cb_list);
1199        mutex_init(&lkb->lkb_cb_mutex);
1200        INIT_WORK(&lkb->lkb_cb_work, dlm_callback_work);
1201
1202        idr_preload(GFP_NOFS);
1203        spin_lock(&ls->ls_lkbidr_spin);
1204        rv = idr_alloc(&ls->ls_lkbidr, lkb, 1, 0, GFP_NOWAIT);
1205        if (rv >= 0)
1206                lkb->lkb_id = rv;
1207        spin_unlock(&ls->ls_lkbidr_spin);
1208        idr_preload_end();
1209
1210        if (rv < 0) {
1211                log_error(ls, "create_lkb idr error %d", rv);
1212                return rv;
1213        }
1214
1215        *lkb_ret = lkb;
1216        return 0;
1217}
1218
1219static int find_lkb(struct dlm_ls *ls, uint32_t lkid, struct dlm_lkb **lkb_ret)
1220{
1221        struct dlm_lkb *lkb;
1222
1223        spin_lock(&ls->ls_lkbidr_spin);
1224        lkb = idr_find(&ls->ls_lkbidr, lkid);
1225        if (lkb)
1226                kref_get(&lkb->lkb_ref);
1227        spin_unlock(&ls->ls_lkbidr_spin);
1228
1229        *lkb_ret = lkb;
1230        return lkb ? 0 : -ENOENT;
1231}
1232
1233static void kill_lkb(struct kref *kref)
1234{
1235        struct dlm_lkb *lkb = container_of(kref, struct dlm_lkb, lkb_ref);
1236
1237        /* All work is done after the return from kref_put() so we
1238           can release the write_lock before the detach_lkb */
1239
1240        DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb););
1241}
1242
1243/* __put_lkb() is used when an lkb may not have an rsb attached to
1244   it so we need to provide the lockspace explicitly */
1245
1246static int __put_lkb(struct dlm_ls *ls, struct dlm_lkb *lkb)
1247{
1248        uint32_t lkid = lkb->lkb_id;
1249
1250        spin_lock(&ls->ls_lkbidr_spin);
1251        if (kref_put(&lkb->lkb_ref, kill_lkb)) {
1252                idr_remove(&ls->ls_lkbidr, lkid);
1253                spin_unlock(&ls->ls_lkbidr_spin);
1254
1255                detach_lkb(lkb);
1256
1257                /* for local/process lkbs, lvbptr points to caller's lksb */
1258                if (lkb->lkb_lvbptr && is_master_copy(lkb))
1259                        dlm_free_lvb(lkb->lkb_lvbptr);
1260                dlm_free_lkb(lkb);
1261                return 1;
1262        } else {
1263                spin_unlock(&ls->ls_lkbidr_spin);
1264                return 0;
1265        }
1266}
1267
1268int dlm_put_lkb(struct dlm_lkb *lkb)
1269{
1270        struct dlm_ls *ls;
1271
1272        DLM_ASSERT(lkb->lkb_resource, dlm_print_lkb(lkb););
1273        DLM_ASSERT(lkb->lkb_resource->res_ls, dlm_print_lkb(lkb););
1274
1275        ls = lkb->lkb_resource->res_ls;
1276        return __put_lkb(ls, lkb);
1277}
1278
1279/* This is only called to add a reference when the code already holds
1280   a valid reference to the lkb, so there's no need for locking. */
1281
1282static inline void hold_lkb(struct dlm_lkb *lkb)
1283{
1284        kref_get(&lkb->lkb_ref);
1285}
1286
1287/* This is called when we need to remove a reference and are certain
1288   it's not the last ref.  e.g. del_lkb is always called between a
1289   find_lkb/put_lkb and is always the inverse of a previous add_lkb.
1290   put_lkb would work fine, but would involve unnecessary locking */
1291
1292static inline void unhold_lkb(struct dlm_lkb *lkb)
1293{
1294        int rv;
1295        rv = kref_put(&lkb->lkb_ref, kill_lkb);
1296        DLM_ASSERT(!rv, dlm_print_lkb(lkb););
1297}
1298
1299static void lkb_add_ordered(struct list_head *new, struct list_head *head,
1300                            int mode)
1301{
1302        struct dlm_lkb *lkb = NULL;
1303
1304        list_for_each_entry(lkb, head, lkb_statequeue)
1305                if (lkb->lkb_rqmode < mode)
1306                        break;
1307
1308        __list_add(new, lkb->lkb_statequeue.prev, &lkb->lkb_statequeue);
1309}
1310
1311/* add/remove lkb to rsb's grant/convert/wait queue */
1312
1313static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status)
1314{
1315        kref_get(&lkb->lkb_ref);
1316
1317        DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb););
1318
1319        lkb->lkb_timestamp = ktime_get();
1320
1321        lkb->lkb_status = status;
1322
1323        switch (status) {
1324        case DLM_LKSTS_WAITING:
1325                if (lkb->lkb_exflags & DLM_LKF_HEADQUE)
1326                        list_add(&lkb->lkb_statequeue, &r->res_waitqueue);
1327                else
1328                        list_add_tail(&lkb->lkb_statequeue, &r->res_waitqueue);
1329                break;
1330        case DLM_LKSTS_GRANTED:
1331                /* convention says granted locks kept in order of grmode */
1332                lkb_add_ordered(&lkb->lkb_statequeue, &r->res_grantqueue,
1333                                lkb->lkb_grmode);
1334                break;
1335        case DLM_LKSTS_CONVERT:
1336                if (lkb->lkb_exflags & DLM_LKF_HEADQUE)
1337                        list_add(&lkb->lkb_statequeue, &r->res_convertqueue);
1338                else
1339                        list_add_tail(&lkb->lkb_statequeue,
1340                                      &r->res_convertqueue);
1341                break;
1342        default:
1343                DLM_ASSERT(0, dlm_print_lkb(lkb); printk("sts=%d\n", status););
1344        }
1345}
1346
1347static void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb)
1348{
1349        lkb->lkb_status = 0;
1350        list_del(&lkb->lkb_statequeue);
1351        unhold_lkb(lkb);
1352}
1353
1354static void move_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int sts)
1355{
1356        hold_lkb(lkb);
1357        del_lkb(r, lkb);
1358        add_lkb(r, lkb, sts);
1359        unhold_lkb(lkb);
1360}
1361
1362static int msg_reply_type(int mstype)
1363{
1364        switch (mstype) {
1365        case DLM_MSG_REQUEST:
1366                return DLM_MSG_REQUEST_REPLY;
1367        case DLM_MSG_CONVERT:
1368                return DLM_MSG_CONVERT_REPLY;
1369        case DLM_MSG_UNLOCK:
1370                return DLM_MSG_UNLOCK_REPLY;
1371        case DLM_MSG_CANCEL:
1372                return DLM_MSG_CANCEL_REPLY;
1373        case DLM_MSG_LOOKUP:
1374                return DLM_MSG_LOOKUP_REPLY;
1375        }
1376        return -1;
1377}
1378
1379static int nodeid_warned(int nodeid, int num_nodes, int *warned)
1380{
1381        int i;
1382
1383        for (i = 0; i < num_nodes; i++) {
1384                if (!warned[i]) {
1385                        warned[i] = nodeid;
1386                        return 0;
1387                }
1388                if (warned[i] == nodeid)
1389                        return 1;
1390        }
1391        return 0;
1392}
1393
1394void dlm_scan_waiters(struct dlm_ls *ls)
1395{
1396        struct dlm_lkb *lkb;
1397        ktime_t zero = ktime_set(0, 0);
1398        s64 us;
1399        s64 debug_maxus = 0;
1400        u32 debug_scanned = 0;
1401        u32 debug_expired = 0;
1402        int num_nodes = 0;
1403        int *warned = NULL;
1404
1405        if (!dlm_config.ci_waitwarn_us)
1406                return;
1407
1408        mutex_lock(&ls->ls_waiters_mutex);
1409
1410        list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) {
1411                if (ktime_equal(lkb->lkb_wait_time, zero))
1412                        continue;
1413
1414                debug_scanned++;
1415
1416                us = ktime_to_us(ktime_sub(ktime_get(), lkb->lkb_wait_time));
1417
1418                if (us < dlm_config.ci_waitwarn_us)
1419                        continue;
1420
1421                lkb->lkb_wait_time = zero;
1422
1423                debug_expired++;
1424                if (us > debug_maxus)
1425                        debug_maxus = us;
1426
1427                if (!num_nodes) {
1428                        num_nodes = ls->ls_num_nodes;
1429                        warned = kzalloc(num_nodes * sizeof(int), GFP_KERNEL);
1430                }
1431                if (!warned)
1432                        continue;
1433                if (nodeid_warned(lkb->lkb_wait_nodeid, num_nodes, warned))
1434                        continue;
1435
1436                log_error(ls, "waitwarn %x %lld %d us check connection to "
1437                          "node %d", lkb->lkb_id, (long long)us,
1438                          dlm_config.ci_waitwarn_us, lkb->lkb_wait_nodeid);
1439        }
1440        mutex_unlock(&ls->ls_waiters_mutex);
1441        kfree(warned);
1442
1443        if (debug_expired)
1444                log_debug(ls, "scan_waiters %u warn %u over %d us max %lld us",
1445                          debug_scanned, debug_expired,
1446                          dlm_config.ci_waitwarn_us, (long long)debug_maxus);
1447}
1448
1449/* add/remove lkb from global waiters list of lkb's waiting for
1450   a reply from a remote node */
1451
1452static int add_to_waiters(struct dlm_lkb *lkb, int mstype, int to_nodeid)
1453{
1454        struct dlm_ls *ls = lkb->lkb_resource->res_ls;
1455        int error = 0;
1456
1457        mutex_lock(&ls->ls_waiters_mutex);
1458
1459        if (is_overlap_unlock(lkb) ||
1460            (is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL))) {
1461                error = -EINVAL;
1462                goto out;
1463        }
1464
1465        if (lkb->lkb_wait_type || is_overlap_cancel(lkb)) {
1466                switch (mstype) {
1467                case DLM_MSG_UNLOCK:
1468                        lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK;
1469                        break;
1470                case DLM_MSG_CANCEL:
1471                        lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL;
1472                        break;
1473                default:
1474                        error = -EBUSY;
1475                        goto out;
1476                }
1477                lkb->lkb_wait_count++;
1478                hold_lkb(lkb);
1479
1480                log_debug(ls, "addwait %x cur %d overlap %d count %d f %x",
1481                          lkb->lkb_id, lkb->lkb_wait_type, mstype,
1482                          lkb->lkb_wait_count, lkb->lkb_flags);
1483                goto out;
1484        }
1485
1486        DLM_ASSERT(!lkb->lkb_wait_count,
1487                   dlm_print_lkb(lkb);
1488                   printk("wait_count %d\n", lkb->lkb_wait_count););
1489
1490        lkb->lkb_wait_count++;
1491        lkb->lkb_wait_type = mstype;
1492        lkb->lkb_wait_time = ktime_get();
1493        lkb->lkb_wait_nodeid = to_nodeid; /* for debugging */
1494        hold_lkb(lkb);
1495        list_add(&lkb->lkb_wait_reply, &ls->ls_waiters);
1496 out:
1497        if (error)
1498                log_error(ls, "addwait error %x %d flags %x %d %d %s",
1499                          lkb->lkb_id, error, lkb->lkb_flags, mstype,
1500                          lkb->lkb_wait_type, lkb->lkb_resource->res_name);
1501        mutex_unlock(&ls->ls_waiters_mutex);
1502        return error;
1503}
1504
1505/* We clear the RESEND flag because we might be taking an lkb off the waiters
1506   list as part of process_requestqueue (e.g. a lookup that has an optimized
1507   request reply on the requestqueue) between dlm_recover_waiters_pre() which
1508   set RESEND and dlm_recover_waiters_post() */
1509
1510static int _remove_from_waiters(struct dlm_lkb *lkb, int mstype,
1511                                struct dlm_message *ms)
1512{
1513        struct dlm_ls *ls = lkb->lkb_resource->res_ls;
1514        int overlap_done = 0;
1515
1516        if (is_overlap_unlock(lkb) && (mstype == DLM_MSG_UNLOCK_REPLY)) {
1517                log_debug(ls, "remwait %x unlock_reply overlap", lkb->lkb_id);
1518                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
1519                overlap_done = 1;
1520                goto out_del;
1521        }
1522
1523        if (is_overlap_cancel(lkb) && (mstype == DLM_MSG_CANCEL_REPLY)) {
1524                log_debug(ls, "remwait %x cancel_reply overlap", lkb->lkb_id);
1525                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
1526                overlap_done = 1;
1527                goto out_del;
1528        }
1529
1530        /* Cancel state was preemptively cleared by a successful convert,
1531           see next comment, nothing to do. */
1532
1533        if ((mstype == DLM_MSG_CANCEL_REPLY) &&
1534            (lkb->lkb_wait_type != DLM_MSG_CANCEL)) {
1535                log_debug(ls, "remwait %x cancel_reply wait_type %d",
1536                          lkb->lkb_id, lkb->lkb_wait_type);
1537                return -1;
1538        }
1539
1540        /* Remove for the convert reply, and premptively remove for the
1541           cancel reply.  A convert has been granted while there's still
1542           an outstanding cancel on it (the cancel is moot and the result
1543           in the cancel reply should be 0).  We preempt the cancel reply
1544           because the app gets the convert result and then can follow up
1545           with another op, like convert.  This subsequent op would see the
1546           lingering state of the cancel and fail with -EBUSY. */
1547
1548        if ((mstype == DLM_MSG_CONVERT_REPLY) &&
1549            (lkb->lkb_wait_type == DLM_MSG_CONVERT) &&
1550            is_overlap_cancel(lkb) && ms && !ms->m_result) {
1551                log_debug(ls, "remwait %x convert_reply zap overlap_cancel",
1552                          lkb->lkb_id);
1553                lkb->lkb_wait_type = 0;
1554                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
1555                lkb->lkb_wait_count--;
1556                goto out_del;
1557        }
1558
1559        /* N.B. type of reply may not always correspond to type of original
1560           msg due to lookup->request optimization, verify others? */
1561
1562        if (lkb->lkb_wait_type) {
1563                lkb->lkb_wait_type = 0;
1564                goto out_del;
1565        }
1566
1567        log_error(ls, "remwait error %x remote %d %x msg %d flags %x no wait",
1568                  lkb->lkb_id, ms ? ms->m_header.h_nodeid : 0, lkb->lkb_remid,
1569                  mstype, lkb->lkb_flags);
1570        return -1;
1571
1572 out_del:
1573        /* the force-unlock/cancel has completed and we haven't recvd a reply
1574           to the op that was in progress prior to the unlock/cancel; we
1575           give up on any reply to the earlier op.  FIXME: not sure when/how
1576           this would happen */
1577
1578        if (overlap_done && lkb->lkb_wait_type) {
1579                log_error(ls, "remwait error %x reply %d wait_type %d overlap",
1580                          lkb->lkb_id, mstype, lkb->lkb_wait_type);
1581                lkb->lkb_wait_count--;
1582                lkb->lkb_wait_type = 0;
1583        }
1584
1585        DLM_ASSERT(lkb->lkb_wait_count, dlm_print_lkb(lkb););
1586
1587        lkb->lkb_flags &= ~DLM_IFL_RESEND;
1588        lkb->lkb_wait_count--;
1589        if (!lkb->lkb_wait_count)
1590                list_del_init(&lkb->lkb_wait_reply);
1591        unhold_lkb(lkb);
1592        return 0;
1593}
1594
1595static int remove_from_waiters(struct dlm_lkb *lkb, int mstype)
1596{
1597        struct dlm_ls *ls = lkb->lkb_resource->res_ls;
1598        int error;
1599
1600        mutex_lock(&ls->ls_waiters_mutex);
1601        error = _remove_from_waiters(lkb, mstype, NULL);
1602        mutex_unlock(&ls->ls_waiters_mutex);
1603        return error;
1604}
1605
1606/* Handles situations where we might be processing a "fake" or "stub" reply in
1607   which we can't try to take waiters_mutex again. */
1608
1609static int remove_from_waiters_ms(struct dlm_lkb *lkb, struct dlm_message *ms)
1610{
1611        struct dlm_ls *ls = lkb->lkb_resource->res_ls;
1612        int error;
1613
1614        if (ms->m_flags != DLM_IFL_STUB_MS)
1615                mutex_lock(&ls->ls_waiters_mutex);
1616        error = _remove_from_waiters(lkb, ms->m_type, ms);
1617        if (ms->m_flags != DLM_IFL_STUB_MS)
1618                mutex_unlock(&ls->ls_waiters_mutex);
1619        return error;
1620}
1621
1622/* If there's an rsb for the same resource being removed, ensure
1623   that the remove message is sent before the new lookup message.
1624   It should be rare to need a delay here, but if not, then it may
1625   be worthwhile to add a proper wait mechanism rather than a delay. */
1626
1627static void wait_pending_remove(struct dlm_rsb *r)
1628{
1629        struct dlm_ls *ls = r->res_ls;
1630 restart:
1631        spin_lock(&ls->ls_remove_spin);
1632        if (ls->ls_remove_len &&
1633            !rsb_cmp(r, ls->ls_remove_name, ls->ls_remove_len)) {
1634                log_debug(ls, "delay lookup for remove dir %d %s",
1635                          r->res_dir_nodeid, r->res_name);
1636                spin_unlock(&ls->ls_remove_spin);
1637                msleep(1);
1638                goto restart;
1639        }
1640        spin_unlock(&ls->ls_remove_spin);
1641}
1642
1643/*
1644 * ls_remove_spin protects ls_remove_name and ls_remove_len which are
1645 * read by other threads in wait_pending_remove.  ls_remove_names
1646 * and ls_remove_lens are only used by the scan thread, so they do
1647 * not need protection.
1648 */
1649
1650static void shrink_bucket(struct dlm_ls *ls, int b)
1651{
1652        struct rb_node *n, *next;
1653        struct dlm_rsb *r;
1654        char *name;
1655        int our_nodeid = dlm_our_nodeid();
1656        int remote_count = 0;
1657        int need_shrink = 0;
1658        int i, len, rv;
1659
1660        memset(&ls->ls_remove_lens, 0, sizeof(int) * DLM_REMOVE_NAMES_MAX);
1661
1662        spin_lock(&ls->ls_rsbtbl[b].lock);
1663
1664        if (!(ls->ls_rsbtbl[b].flags & DLM_RTF_SHRINK)) {
1665                spin_unlock(&ls->ls_rsbtbl[b].lock);
1666                return;
1667        }
1668
1669        for (n = rb_first(&ls->ls_rsbtbl[b].toss); n; n = next) {
1670                next = rb_next(n);
1671                r = rb_entry(n, struct dlm_rsb, res_hashnode);
1672
1673                /* If we're the directory record for this rsb, and
1674                   we're not the master of it, then we need to wait
1675                   for the master node to send us a dir remove for
1676                   before removing the dir record. */
1677
1678                if (!dlm_no_directory(ls) &&
1679                    (r->res_master_nodeid != our_nodeid) &&
1680                    (dlm_dir_nodeid(r) == our_nodeid)) {
1681                        continue;
1682                }
1683
1684                need_shrink = 1;
1685
1686                if (!time_after_eq(jiffies, r->res_toss_time +
1687                                   dlm_config.ci_toss_secs * HZ)) {
1688                        continue;
1689                }
1690
1691                if (!dlm_no_directory(ls) &&
1692                    (r->res_master_nodeid == our_nodeid) &&
1693                    (dlm_dir_nodeid(r) != our_nodeid)) {
1694
1695                        /* We're the master of this rsb but we're not
1696                           the directory record, so we need to tell the
1697                           dir node to remove the dir record. */
1698
1699                        ls->ls_remove_lens[remote_count] = r->res_length;
1700                        memcpy(ls->ls_remove_names[remote_count], r->res_name,
1701                               DLM_RESNAME_MAXLEN);
1702                        remote_count++;
1703
1704                        if (remote_count >= DLM_REMOVE_NAMES_MAX)
1705                                break;
1706                        continue;
1707                }
1708
1709                if (!kref_put(&r->res_ref, kill_rsb)) {
1710                        log_error(ls, "tossed rsb in use %s", r->res_name);
1711                        continue;
1712                }
1713
1714                rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss);
1715                dlm_free_rsb(r);
1716        }
1717
1718        if (need_shrink)
1719                ls->ls_rsbtbl[b].flags |= DLM_RTF_SHRINK;
1720        else
1721                ls->ls_rsbtbl[b].flags &= ~DLM_RTF_SHRINK;
1722        spin_unlock(&ls->ls_rsbtbl[b].lock);
1723
1724        /*
1725         * While searching for rsb's to free, we found some that require
1726         * remote removal.  We leave them in place and find them again here
1727         * so there is a very small gap between removing them from the toss
1728         * list and sending the removal.  Keeping this gap small is
1729         * important to keep us (the master node) from being out of sync
1730         * with the remote dir node for very long.
1731         *
1732         * From the time the rsb is removed from toss until just after
1733         * send_remove, the rsb name is saved in ls_remove_name.  A new
1734         * lookup checks this to ensure that a new lookup message for the
1735         * same resource name is not sent just before the remove message.
1736         */
1737
1738        for (i = 0; i < remote_count; i++) {
1739                name = ls->ls_remove_names[i];
1740                len = ls->ls_remove_lens[i];
1741
1742                spin_lock(&ls->ls_rsbtbl[b].lock);
1743                rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r);
1744                if (rv) {
1745                        spin_unlock(&ls->ls_rsbtbl[b].lock);
1746                        log_debug(ls, "remove_name not toss %s", name);
1747                        continue;
1748                }
1749
1750                if (r->res_master_nodeid != our_nodeid) {
1751                        spin_unlock(&ls->ls_rsbtbl[b].lock);
1752                        log_debug(ls, "remove_name master %d dir %d our %d %s",
1753                                  r->res_master_nodeid, r->res_dir_nodeid,
1754                                  our_nodeid, name);
1755                        continue;
1756                }
1757
1758                if (r->res_dir_nodeid == our_nodeid) {
1759                        /* should never happen */
1760                        spin_unlock(&ls->ls_rsbtbl[b].lock);
1761                        log_error(ls, "remove_name dir %d master %d our %d %s",
1762                                  r->res_dir_nodeid, r->res_master_nodeid,
1763                                  our_nodeid, name);
1764                        continue;
1765                }
1766
1767                if (!time_after_eq(jiffies, r->res_toss_time +
1768                                   dlm_config.ci_toss_secs * HZ)) {
1769                        spin_unlock(&ls->ls_rsbtbl[b].lock);
1770                        log_debug(ls, "remove_name toss_time %lu now %lu %s",
1771                                  r->res_toss_time, jiffies, name);
1772                        continue;
1773                }
1774
1775                if (!kref_put(&r->res_ref, kill_rsb)) {
1776                        spin_unlock(&ls->ls_rsbtbl[b].lock);
1777                        log_error(ls, "remove_name in use %s", name);
1778                        continue;
1779                }
1780
1781                rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss);
1782
1783                /* block lookup of same name until we've sent remove */
1784                spin_lock(&ls->ls_remove_spin);
1785                ls->ls_remove_len = len;
1786                memcpy(ls->ls_remove_name, name, DLM_RESNAME_MAXLEN);
1787                spin_unlock(&ls->ls_remove_spin);
1788                spin_unlock(&ls->ls_rsbtbl[b].lock);
1789
1790                send_remove(r);
1791
1792                /* allow lookup of name again */
1793                spin_lock(&ls->ls_remove_spin);
1794                ls->ls_remove_len = 0;
1795                memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN);
1796                spin_unlock(&ls->ls_remove_spin);
1797
1798                dlm_free_rsb(r);
1799        }
1800}
1801
1802void dlm_scan_rsbs(struct dlm_ls *ls)
1803{
1804        int i;
1805
1806        for (i = 0; i < ls->ls_rsbtbl_size; i++) {
1807                shrink_bucket(ls, i);
1808                if (dlm_locking_stopped(ls))
1809                        break;
1810                cond_resched();
1811        }
1812}
1813
1814static void add_timeout(struct dlm_lkb *lkb)
1815{
1816        struct dlm_ls *ls = lkb->lkb_resource->res_ls;
1817
1818        if (is_master_copy(lkb))
1819                return;
1820
1821        if (test_bit(LSFL_TIMEWARN, &ls->ls_flags) &&
1822            !(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) {
1823                lkb->lkb_flags |= DLM_IFL_WATCH_TIMEWARN;
1824                goto add_it;
1825        }
1826        if (lkb->lkb_exflags & DLM_LKF_TIMEOUT)
1827                goto add_it;
1828        return;
1829
1830 add_it:
1831        DLM_ASSERT(list_empty(&lkb->lkb_time_list), dlm_print_lkb(lkb););
1832        mutex_lock(&ls->ls_timeout_mutex);
1833        hold_lkb(lkb);
1834        list_add_tail(&lkb->lkb_time_list, &ls->ls_timeout);
1835        mutex_unlock(&ls->ls_timeout_mutex);
1836}
1837
1838static void del_timeout(struct dlm_lkb *lkb)
1839{
1840        struct dlm_ls *ls = lkb->lkb_resource->res_ls;
1841
1842        mutex_lock(&ls->ls_timeout_mutex);
1843        if (!list_empty(&lkb->lkb_time_list)) {
1844                list_del_init(&lkb->lkb_time_list);
1845                unhold_lkb(lkb);
1846        }
1847        mutex_unlock(&ls->ls_timeout_mutex);
1848}
1849
1850/* FIXME: is it safe to look at lkb_exflags, lkb_flags, lkb_timestamp, and
1851   lkb_lksb_timeout without lock_rsb?  Note: we can't lock timeout_mutex
1852   and then lock rsb because of lock ordering in add_timeout.  We may need
1853   to specify some special timeout-related bits in the lkb that are just to
1854   be accessed under the timeout_mutex. */
1855
1856void dlm_scan_timeout(struct dlm_ls *ls)
1857{
1858        struct dlm_rsb *r;
1859        struct dlm_lkb *lkb;
1860        int do_cancel, do_warn;
1861        s64 wait_us;
1862
1863        for (;;) {
1864                if (dlm_locking_stopped(ls))
1865                        break;
1866
1867                do_cancel = 0;
1868                do_warn = 0;
1869                mutex_lock(&ls->ls_timeout_mutex);
1870                list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list) {
1871
1872                        wait_us = ktime_to_us(ktime_sub(ktime_get(),
1873                                                        lkb->lkb_timestamp));
1874
1875                        if ((lkb->lkb_exflags & DLM_LKF_TIMEOUT) &&
1876                            wait_us >= (lkb->lkb_timeout_cs * 10000))
1877                                do_cancel = 1;
1878
1879                        if ((lkb->lkb_flags & DLM_IFL_WATCH_TIMEWARN) &&
1880                            wait_us >= dlm_config.ci_timewarn_cs * 10000)
1881                                do_warn = 1;
1882
1883                        if (!do_cancel && !do_warn)
1884                                continue;
1885                        hold_lkb(lkb);
1886                        break;
1887                }
1888                mutex_unlock(&ls->ls_timeout_mutex);
1889
1890                if (!do_cancel && !do_warn)
1891                        break;
1892
1893                r = lkb->lkb_resource;
1894                hold_rsb(r);
1895                lock_rsb(r);
1896
1897                if (do_warn) {
1898                        /* clear flag so we only warn once */
1899                        lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN;
1900                        if (!(lkb->lkb_exflags & DLM_LKF_TIMEOUT))
1901                                del_timeout(lkb);
1902                        dlm_timeout_warn(lkb);
1903                }
1904
1905                if (do_cancel) {
1906                        log_debug(ls, "timeout cancel %x node %d %s",
1907                                  lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
1908                        lkb->lkb_flags &= ~DLM_IFL_WATCH_TIMEWARN;
1909                        lkb->lkb_flags |= DLM_IFL_TIMEOUT_CANCEL;
1910                        del_timeout(lkb);
1911                        _cancel_lock(r, lkb);
1912                }
1913
1914                unlock_rsb(r);
1915                unhold_rsb(r);
1916                dlm_put_lkb(lkb);
1917        }
1918}
1919
1920/* This is only called by dlm_recoverd, and we rely on dlm_ls_stop() stopping
1921   dlm_recoverd before checking/setting ls_recover_begin. */
1922
1923void dlm_adjust_timeouts(struct dlm_ls *ls)
1924{
1925        struct dlm_lkb *lkb;
1926        u64 adj_us = jiffies_to_usecs(jiffies - ls->ls_recover_begin);
1927
1928        ls->ls_recover_begin = 0;
1929        mutex_lock(&ls->ls_timeout_mutex);
1930        list_for_each_entry(lkb, &ls->ls_timeout, lkb_time_list)
1931                lkb->lkb_timestamp = ktime_add_us(lkb->lkb_timestamp, adj_us);
1932        mutex_unlock(&ls->ls_timeout_mutex);
1933
1934        if (!dlm_config.ci_waitwarn_us)
1935                return;
1936
1937        mutex_lock(&ls->ls_waiters_mutex);
1938        list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) {
1939                if (ktime_to_us(lkb->lkb_wait_time))
1940                        lkb->lkb_wait_time = ktime_get();
1941        }
1942        mutex_unlock(&ls->ls_waiters_mutex);
1943}
1944
1945/* lkb is master or local copy */
1946
1947static void set_lvb_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
1948{
1949        int b, len = r->res_ls->ls_lvblen;
1950
1951        /* b=1 lvb returned to caller
1952           b=0 lvb written to rsb or invalidated
1953           b=-1 do nothing */
1954
1955        b =  dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1];
1956
1957        if (b == 1) {
1958                if (!lkb->lkb_lvbptr)
1959                        return;
1960
1961                if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
1962                        return;
1963
1964                if (!r->res_lvbptr)
1965                        return;
1966
1967                memcpy(lkb->lkb_lvbptr, r->res_lvbptr, len);
1968                lkb->lkb_lvbseq = r->res_lvbseq;
1969
1970        } else if (b == 0) {
1971                if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) {
1972                        rsb_set_flag(r, RSB_VALNOTVALID);
1973                        return;
1974                }
1975
1976                if (!lkb->lkb_lvbptr)
1977                        return;
1978
1979                if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
1980                        return;
1981
1982                if (!r->res_lvbptr)
1983                        r->res_lvbptr = dlm_allocate_lvb(r->res_ls);
1984
1985                if (!r->res_lvbptr)
1986                        return;
1987
1988                memcpy(r->res_lvbptr, lkb->lkb_lvbptr, len);
1989                r->res_lvbseq++;
1990                lkb->lkb_lvbseq = r->res_lvbseq;
1991                rsb_clear_flag(r, RSB_VALNOTVALID);
1992        }
1993
1994        if (rsb_flag(r, RSB_VALNOTVALID))
1995                lkb->lkb_sbflags |= DLM_SBF_VALNOTVALID;
1996}
1997
1998static void set_lvb_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb)
1999{
2000        if (lkb->lkb_grmode < DLM_LOCK_PW)
2001                return;
2002
2003        if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) {
2004                rsb_set_flag(r, RSB_VALNOTVALID);
2005                return;
2006        }
2007
2008        if (!lkb->lkb_lvbptr)
2009                return;
2010
2011        if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
2012                return;
2013
2014        if (!r->res_lvbptr)
2015                r->res_lvbptr = dlm_allocate_lvb(r->res_ls);
2016
2017        if (!r->res_lvbptr)
2018                return;
2019
2020        memcpy(r->res_lvbptr, lkb->lkb_lvbptr, r->res_ls->ls_lvblen);
2021        r->res_lvbseq++;
2022        rsb_clear_flag(r, RSB_VALNOTVALID);
2023}
2024
2025/* lkb is process copy (pc) */
2026
2027static void set_lvb_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb,
2028                            struct dlm_message *ms)
2029{
2030        int b;
2031
2032        if (!lkb->lkb_lvbptr)
2033                return;
2034
2035        if (!(lkb->lkb_exflags & DLM_LKF_VALBLK))
2036                return;
2037
2038        b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1];
2039        if (b == 1) {
2040                int len = receive_extralen(ms);
2041                if (len > r->res_ls->ls_lvblen)
2042                        len = r->res_ls->ls_lvblen;
2043                memcpy(lkb->lkb_lvbptr, ms->m_extra, len);
2044                lkb->lkb_lvbseq = ms->m_lvbseq;
2045        }
2046}
2047
2048/* Manipulate lkb's on rsb's convert/granted/waiting queues
2049   remove_lock -- used for unlock, removes lkb from granted
2050   revert_lock -- used for cancel, moves lkb from convert to granted
2051   grant_lock  -- used for request and convert, adds lkb to granted or
2052                  moves lkb from convert or waiting to granted
2053
2054   Each of these is used for master or local copy lkb's.  There is
2055   also a _pc() variation used to make the corresponding change on
2056   a process copy (pc) lkb. */
2057
2058static void _remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
2059{
2060        del_lkb(r, lkb);
2061        lkb->lkb_grmode = DLM_LOCK_IV;
2062        /* this unhold undoes the original ref from create_lkb()
2063           so this leads to the lkb being freed */
2064        unhold_lkb(lkb);
2065}
2066
2067static void remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
2068{
2069        set_lvb_unlock(r, lkb);
2070        _remove_lock(r, lkb);
2071}
2072
2073static void remove_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb)
2074{
2075        _remove_lock(r, lkb);
2076}
2077
2078/* returns: 0 did nothing
2079            1 moved lock to granted
2080           -1 removed lock */
2081
2082static int revert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
2083{
2084        int rv = 0;
2085
2086        lkb->lkb_rqmode = DLM_LOCK_IV;
2087
2088        switch (lkb->lkb_status) {
2089        case DLM_LKSTS_GRANTED:
2090                break;
2091        case DLM_LKSTS_CONVERT:
2092                move_lkb(r, lkb, DLM_LKSTS_GRANTED);
2093                rv = 1;
2094                break;
2095        case DLM_LKSTS_WAITING:
2096                del_lkb(r, lkb);
2097                lkb->lkb_grmode = DLM_LOCK_IV;
2098                /* this unhold undoes the original ref from create_lkb()
2099                   so this leads to the lkb being freed */
2100                unhold_lkb(lkb);
2101                rv = -1;
2102                break;
2103        default:
2104                log_print("invalid status for revert %d", lkb->lkb_status);
2105        }
2106        return rv;
2107}
2108
2109static int revert_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb)
2110{
2111        return revert_lock(r, lkb);
2112}
2113
2114static void _grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
2115{
2116        if (lkb->lkb_grmode != lkb->lkb_rqmode) {
2117                lkb->lkb_grmode = lkb->lkb_rqmode;
2118                if (lkb->lkb_status)
2119                        move_lkb(r, lkb, DLM_LKSTS_GRANTED);
2120                else
2121                        add_lkb(r, lkb, DLM_LKSTS_GRANTED);
2122        }
2123
2124        lkb->lkb_rqmode = DLM_LOCK_IV;
2125        lkb->lkb_highbast = 0;
2126}
2127
2128static void grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
2129{
2130        set_lvb_lock(r, lkb);
2131        _grant_lock(r, lkb);
2132}
2133
2134static void grant_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb,
2135                          struct dlm_message *ms)
2136{
2137        set_lvb_lock_pc(r, lkb, ms);
2138        _grant_lock(r, lkb);
2139}
2140
2141/* called by grant_pending_locks() which means an async grant message must
2142   be sent to the requesting node in addition to granting the lock if the
2143   lkb belongs to a remote node. */
2144
2145static void grant_lock_pending(struct dlm_rsb *r, struct dlm_lkb *lkb)
2146{
2147        grant_lock(r, lkb);
2148        if (is_master_copy(lkb))
2149                send_grant(r, lkb);
2150        else
2151                queue_cast(r, lkb, 0);
2152}
2153
2154/* The special CONVDEADLK, ALTPR and ALTCW flags allow the master to
2155   change the granted/requested modes.  We're munging things accordingly in
2156   the process copy.
2157   CONVDEADLK: our grmode may have been forced down to NL to resolve a
2158   conversion deadlock
2159   ALTPR/ALTCW: our rqmode may have been changed to PR or CW to become
2160   compatible with other granted locks */
2161
2162static void munge_demoted(struct dlm_lkb *lkb)
2163{
2164        if (lkb->lkb_rqmode == DLM_LOCK_IV || lkb->lkb_grmode == DLM_LOCK_IV) {
2165                log_print("munge_demoted %x invalid modes gr %d rq %d",
2166                          lkb->lkb_id, lkb->lkb_grmode, lkb->lkb_rqmode);
2167                return;
2168        }
2169
2170        lkb->lkb_grmode = DLM_LOCK_NL;
2171}
2172
2173static void munge_altmode(struct dlm_lkb *lkb, struct dlm_message *ms)
2174{
2175        if (ms->m_type != DLM_MSG_REQUEST_REPLY &&
2176            ms->m_type != DLM_MSG_GRANT) {
2177                log_print("munge_altmode %x invalid reply type %d",
2178                          lkb->lkb_id, ms->m_type);
2179                return;
2180        }
2181
2182        if (lkb->lkb_exflags & DLM_LKF_ALTPR)
2183                lkb->lkb_rqmode = DLM_LOCK_PR;
2184        else if (lkb->lkb_exflags & DLM_LKF_ALTCW)
2185                lkb->lkb_rqmode = DLM_LOCK_CW;
2186        else {
2187                log_print("munge_altmode invalid exflags %x", lkb->lkb_exflags);
2188                dlm_print_lkb(lkb);
2189        }
2190}
2191
2192static inline int first_in_list(struct dlm_lkb *lkb, struct list_head *head)
2193{
2194        struct dlm_lkb *first = list_entry(head->next, struct dlm_lkb,
2195                                           lkb_statequeue);
2196        if (lkb->lkb_id == first->lkb_id)
2197                return 1;
2198
2199        return 0;
2200}
2201
2202/* Check if the given lkb conflicts with another lkb on the queue. */
2203
2204static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb)
2205{
2206        struct dlm_lkb *this;
2207
2208        list_for_each_entry(this, head, lkb_statequeue) {
2209                if (this == lkb)
2210                        continue;
2211                if (!modes_compat(this, lkb))
2212                        return 1;
2213        }
2214        return 0;
2215}
2216
2217/*
2218 * "A conversion deadlock arises with a pair of lock requests in the converting
2219 * queue for one resource.  The granted mode of each lock blocks the requested
2220 * mode of the other lock."
2221 *
2222 * Part 2: if the granted mode of lkb is preventing an earlier lkb in the
2223 * convert queue from being granted, then deadlk/demote lkb.
2224 *
2225 * Example:
2226 * Granted Queue: empty
2227 * Convert Queue: NL->EX (first lock)
2228 *                PR->EX (second lock)
2229 *
2230 * The first lock can't be granted because of the granted mode of the second
2231 * lock and the second lock can't be granted because it's not first in the
2232 * list.  We either cancel lkb's conversion (PR->EX) and return EDEADLK, or we
2233 * demote the granted mode of lkb (from PR to NL) if it has the CONVDEADLK
2234 * flag set and return DEMOTED in the lksb flags.
2235 *
2236 * Originally, this function detected conv-deadlk in a more limited scope:
2237 * - if !modes_compat(lkb1, lkb2) && !modes_compat(lkb2, lkb1), or
2238 * - if lkb1 was the first entry in the queue (not just earlier), and was
2239 *   blocked by the granted mode of lkb2, and there was nothing on the
2240 *   granted queue preventing lkb1 from being granted immediately, i.e.
2241 *   lkb2 was the only thing preventing lkb1 from being granted.
2242 *
2243 * That second condition meant we'd only say there was conv-deadlk if
2244 * resolving it (by demotion) would lead to the first lock on the convert
2245 * queue being granted right away.  It allowed conversion deadlocks to exist
2246 * between locks on the convert queue while they couldn't be granted anyway.
2247 *
2248 * Now, we detect and take action on conversion deadlocks immediately when
2249 * they're created, even if they may not be immediately consequential.  If
2250 * lkb1 exists anywhere in the convert queue and lkb2 comes in with a granted
2251 * mode that would prevent lkb1's conversion from being granted, we do a
2252 * deadlk/demote on lkb2 right away and don't let it onto the convert queue.
2253 * I think this means that the lkb_is_ahead condition below should always
2254 * be zero, i.e. there will never be conv-deadlk between two locks that are
2255 * both already on the convert queue.
2256 */
2257
2258static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2)
2259{
2260        struct dlm_lkb *lkb1;
2261        int lkb_is_ahead = 0;
2262
2263        list_for_each_entry(lkb1, &r->res_convertqueue, lkb_statequeue) {
2264                if (lkb1 == lkb2) {
2265                        lkb_is_ahead = 1;
2266                        continue;
2267                }
2268
2269                if (!lkb_is_ahead) {
2270                        if (!modes_compat(lkb2, lkb1))
2271                                return 1;
2272                } else {
2273                        if (!modes_compat(lkb2, lkb1) &&
2274                            !modes_compat(lkb1, lkb2))
2275                                return 1;
2276                }
2277        }
2278        return 0;
2279}
2280
2281/*
2282 * Return 1 if the lock can be granted, 0 otherwise.
2283 * Also detect and resolve conversion deadlocks.
2284 *
2285 * lkb is the lock to be granted
2286 *
2287 * now is 1 if the function is being called in the context of the
2288 * immediate request, it is 0 if called later, after the lock has been
2289 * queued.
2290 *
2291 * recover is 1 if dlm_recover_grant() is trying to grant conversions
2292 * after recovery.
2293 *
2294 * References are from chapter 6 of "VAXcluster Principles" by Roy Davis
2295 */
2296
2297static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now,
2298                           int recover)
2299{
2300        int8_t conv = (lkb->lkb_grmode != DLM_LOCK_IV);
2301
2302        /*
2303         * 6-10: Version 5.4 introduced an option to address the phenomenon of
2304         * a new request for a NL mode lock being blocked.
2305         *
2306         * 6-11: If the optional EXPEDITE flag is used with the new NL mode
2307         * request, then it would be granted.  In essence, the use of this flag
2308         * tells the Lock Manager to expedite theis request by not considering
2309         * what may be in the CONVERTING or WAITING queues...  As of this
2310         * writing, the EXPEDITE flag can be used only with new requests for NL
2311         * mode locks.  This flag is not valid for conversion requests.
2312         *
2313         * A shortcut.  Earlier checks return an error if EXPEDITE is used in a
2314         * conversion or used with a non-NL requested mode.  We also know an
2315         * EXPEDITE request is always granted immediately, so now must always
2316         * be 1.  The full condition to grant an expedite request: (now &&
2317         * !conv && lkb->rqmode == DLM_LOCK_NL && (flags & EXPEDITE)) can
2318         * therefore be shortened to just checking the flag.
2319         */
2320
2321        if (lkb->lkb_exflags & DLM_LKF_EXPEDITE)
2322                return 1;
2323
2324        /*
2325         * A shortcut. Without this, !queue_conflict(grantqueue, lkb) would be
2326         * added to the remaining conditions.
2327         */
2328
2329        if (queue_conflict(&r->res_grantqueue, lkb))
2330                return 0;
2331
2332        /*
2333         * 6-3: By default, a conversion request is immediately granted if the
2334         * requested mode is compatible with the modes of all other granted
2335         * locks
2336         */
2337
2338        if (queue_conflict(&r->res_convertqueue, lkb))
2339                return 0;
2340
2341        /*
2342         * The RECOVER_GRANT flag means dlm_recover_grant() is granting
2343         * locks for a recovered rsb, on which lkb's have been rebuilt.
2344         * The lkb's may have been rebuilt on the queues in a different
2345         * order than they were in on the previous master.  So, granting
2346         * queued conversions in order after recovery doesn't make sense
2347         * since the order hasn't been preserved anyway.  The new order
2348         * could also have created a new "in place" conversion deadlock.
2349         * (e.g. old, failed master held granted EX, with PR->EX, NL->EX.
2350         * After recovery, there would be no granted locks, and possibly
2351         * NL->EX, PR->EX, an in-place conversion deadlock.)  So, after
2352         * recovery, grant conversions without considering order.
2353         */
2354
2355        if (conv && recover)
2356                return 1;
2357
2358        /*
2359         * 6-5: But the default algorithm for deciding whether to grant or
2360         * queue conversion requests does not by itself guarantee that such
2361         * requests are serviced on a "first come first serve" basis.  This, in
2362         * turn, can lead to a phenomenon known as "indefinate postponement".
2363         *
2364         * 6-7: This issue is dealt with by using the optional QUECVT flag with
2365         * the system service employed to request a lock conversion.  This flag
2366         * forces certain conversion requests to be queued, even if they are
2367         * compatible with the granted modes of other locks on the same
2368         * resource.  Thus, the use of this flag results in conversion requests
2369         * being ordered on a "first come first servce" basis.
2370         *
2371         * DCT: This condition is all about new conversions being able to occur
2372         * "in place" while the lock remains on the granted queue (assuming
2373         * nothing else conflicts.)  IOW if QUECVT isn't set, a conversion
2374         * doesn't _have_ to go onto the convert queue where it's processed in
2375         * order.  The "now" variable is necessary to distinguish converts
2376         * being received and processed for the first time now, because once a
2377         * convert is moved to the conversion queue the condition below applies
2378         * requiring fifo granting.
2379         */
2380
2381        if (now && conv && !(lkb->lkb_exflags & DLM_LKF_QUECVT))
2382                return 1;
2383
2384        /*
2385         * Even if the convert is compat with all granted locks,
2386         * QUECVT forces it behind other locks on the convert queue.
2387         */
2388
2389        if (now && conv && (lkb->lkb_exflags & DLM_LKF_QUECVT)) {
2390                if (list_empty(&r->res_convertqueue))
2391                        return 1;
2392                else
2393                        return 0;
2394        }
2395
2396        /*
2397         * The NOORDER flag is set to avoid the standard vms rules on grant
2398         * order.
2399         */
2400
2401        if (lkb->lkb_exflags & DLM_LKF_NOORDER)
2402                return 1;
2403
2404        /*
2405         * 6-3: Once in that queue [CONVERTING], a conversion request cannot be
2406         * granted until all other conversion requests ahead of it are granted
2407         * and/or canceled.
2408         */
2409
2410        if (!now && conv && first_in_list(lkb, &r->res_convertqueue))
2411                return 1;
2412
2413        /*
2414         * 6-4: By default, a new request is immediately granted only if all
2415         * three of the following conditions are satisfied when the request is
2416         * issued:
2417         * - The queue of ungranted conversion requests for the resource is
2418         *   empty.
2419         * - The queue of ungranted new requests for the resource is empty.
2420         * - The mode of the new request is compatible with the most
2421         *   restrictive mode of all granted locks on the resource.
2422         */
2423
2424        if (now && !conv && list_empty(&r->res_convertqueue) &&
2425            list_empty(&r->res_waitqueue))
2426                return 1;
2427
2428        /*
2429         * 6-4: Once a lock request is in the queue of ungranted new requests,
2430         * it cannot be granted until the queue of ungranted conversion
2431         * requests is empty, all ungranted new requests ahead of it are
2432         * granted and/or canceled, and it is compatible with the granted mode
2433         * of the most restrictive lock granted on the resource.
2434         */
2435
2436        if (!now && !conv && list_empty(&r->res_convertqueue) &&
2437            first_in_list(lkb, &r->res_waitqueue))
2438                return 1;
2439
2440        return 0;
2441}
2442
2443static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now,
2444                          int recover, int *err)
2445{
2446        int rv;
2447        int8_t alt = 0, rqmode = lkb->lkb_rqmode;
2448        int8_t is_convert = (lkb->lkb_grmode != DLM_LOCK_IV);
2449
2450        if (err)
2451                *err = 0;
2452
2453        rv = _can_be_granted(r, lkb, now, recover);
2454        if (rv)
2455                goto out;
2456
2457        /*
2458         * The CONVDEADLK flag is non-standard and tells the dlm to resolve
2459         * conversion deadlocks by demoting grmode to NL, otherwise the dlm
2460         * cancels one of the locks.
2461         */
2462
2463        if (is_convert && can_be_queued(lkb) &&
2464            conversion_deadlock_detect(r, lkb)) {
2465                if (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) {
2466                        lkb->lkb_grmode = DLM_LOCK_NL;
2467                        lkb->lkb_sbflags |= DLM_SBF_DEMOTED;
2468                } else if (!(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) {
2469                        if (err)
2470                                *err = -EDEADLK;
2471                        else {
2472                                log_print("can_be_granted deadlock %x now %d",
2473                                          lkb->lkb_id, now);
2474                                dlm_dump_rsb(r);
2475                        }
2476                }
2477                goto out;
2478        }
2479
2480        /*
2481         * The ALTPR and ALTCW flags are non-standard and tell the dlm to try
2482         * to grant a request in a mode other than the normal rqmode.  It's a
2483         * simple way to provide a big optimization to applications that can
2484         * use them.
2485         */
2486
2487        if (rqmode != DLM_LOCK_PR && (lkb->lkb_exflags & DLM_LKF_ALTPR))
2488                alt = DLM_LOCK_PR;
2489        else if (rqmode != DLM_LOCK_CW && (lkb->lkb_exflags & DLM_LKF_ALTCW))
2490                alt = DLM_LOCK_CW;
2491
2492        if (alt) {
2493                lkb->lkb_rqmode = alt;
2494                rv = _can_be_granted(r, lkb, now, 0);
2495                if (rv)
2496                        lkb->lkb_sbflags |= DLM_SBF_ALTMODE;
2497                else
2498                        lkb->lkb_rqmode = rqmode;
2499        }
2500 out:
2501        return rv;
2502}
2503
2504/* FIXME: I don't think that can_be_granted() can/will demote or find deadlock
2505   for locks pending on the convert list.  Once verified (watch for these
2506   log_prints), we should be able to just call _can_be_granted() and not
2507   bother with the demote/deadlk cases here (and there's no easy way to deal
2508   with a deadlk here, we'd have to generate something like grant_lock with
2509   the deadlk error.) */
2510
2511/* Returns the highest requested mode of all blocked conversions; sets
2512   cw if there's a blocked conversion to DLM_LOCK_CW. */
2513
2514static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw,
2515                                 unsigned int *count)
2516{
2517        struct dlm_lkb *lkb, *s;
2518        int recover = rsb_flag(r, RSB_RECOVER_GRANT);
2519        int hi, demoted, quit, grant_restart, demote_restart;
2520        int deadlk;
2521
2522        quit = 0;
2523 restart:
2524        grant_restart = 0;
2525        demote_restart = 0;
2526        hi = DLM_LOCK_IV;
2527
2528        list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) {
2529                demoted = is_demoted(lkb);
2530                deadlk = 0;
2531
2532                if (can_be_granted(r, lkb, 0, recover, &deadlk)) {
2533                        grant_lock_pending(r, lkb);
2534                        grant_restart = 1;
2535                        if (count)
2536                                (*count)++;
2537                        continue;
2538                }
2539
2540                if (!demoted && is_demoted(lkb)) {
2541                        log_print("WARN: pending demoted %x node %d %s",
2542                                  lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
2543                        demote_restart = 1;
2544                        continue;
2545                }
2546
2547                if (deadlk) {
2548                        log_print("WARN: pending deadlock %x node %d %s",
2549                                  lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
2550                        dlm_dump_rsb(r);
2551                        continue;
2552                }
2553
2554                hi = max_t(int, lkb->lkb_rqmode, hi);
2555
2556                if (cw && lkb->lkb_rqmode == DLM_LOCK_CW)
2557                        *cw = 1;
2558        }
2559
2560        if (grant_restart)
2561                goto restart;
2562        if (demote_restart && !quit) {
2563                quit = 1;
2564                goto restart;
2565        }
2566
2567        return max_t(int, high, hi);
2568}
2569
2570static int grant_pending_wait(struct dlm_rsb *r, int high, int *cw,
2571                              unsigned int *count)
2572{
2573        struct dlm_lkb *lkb, *s;
2574
2575        list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) {
2576                if (can_be_granted(r, lkb, 0, 0, NULL)) {
2577                        grant_lock_pending(r, lkb);
2578                        if (count)
2579                                (*count)++;
2580                } else {
2581                        high = max_t(int, lkb->lkb_rqmode, high);
2582                        if (lkb->lkb_rqmode == DLM_LOCK_CW)
2583                                *cw = 1;
2584                }
2585        }
2586
2587        return high;
2588}
2589
2590/* cw of 1 means there's a lock with a rqmode of DLM_LOCK_CW that's blocked
2591   on either the convert or waiting queue.
2592   high is the largest rqmode of all locks blocked on the convert or
2593   waiting queue. */
2594
2595static int lock_requires_bast(struct dlm_lkb *gr, int high, int cw)
2596{
2597        if (gr->lkb_grmode == DLM_LOCK_PR && cw) {
2598                if (gr->lkb_highbast < DLM_LOCK_EX)
2599                        return 1;
2600                return 0;
2601        }
2602
2603        if (gr->lkb_highbast < high &&
2604            !__dlm_compat_matrix[gr->lkb_grmode+1][high+1])
2605                return 1;
2606        return 0;
2607}
2608
2609static void grant_pending_locks(struct dlm_rsb *r, unsigned int *count)
2610{
2611        struct dlm_lkb *lkb, *s;
2612        int high = DLM_LOCK_IV;
2613        int cw = 0;
2614
2615        if (!is_master(r)) {
2616                log_print("grant_pending_locks r nodeid %d", r->res_nodeid);
2617                dlm_dump_rsb(r);
2618                return;
2619        }
2620
2621        high = grant_pending_convert(r, high, &cw, count);
2622        high = grant_pending_wait(r, high, &cw, count);
2623
2624        if (high == DLM_LOCK_IV)
2625                return;
2626
2627        /*
2628         * If there are locks left on the wait/convert queue then send blocking
2629         * ASTs to granted locks based on the largest requested mode (high)
2630         * found above.
2631         */
2632
2633        list_for_each_entry_safe(lkb, s, &r->res_grantqueue, lkb_statequeue) {
2634                if (lkb->lkb_bastfn && lock_requires_bast(lkb, high, cw)) {
2635                        if (cw && high == DLM_LOCK_PR &&
2636                            lkb->lkb_grmode == DLM_LOCK_PR)
2637                                queue_bast(r, lkb, DLM_LOCK_CW);
2638                        else
2639                                queue_bast(r, lkb, high);
2640                        lkb->lkb_highbast = high;
2641                }
2642        }
2643}
2644
2645static int modes_require_bast(struct dlm_lkb *gr, struct dlm_lkb *rq)
2646{
2647        if ((gr->lkb_grmode == DLM_LOCK_PR && rq->lkb_rqmode == DLM_LOCK_CW) ||
2648            (gr->lkb_grmode == DLM_LOCK_CW && rq->lkb_rqmode == DLM_LOCK_PR)) {
2649                if (gr->lkb_highbast < DLM_LOCK_EX)
2650                        return 1;
2651                return 0;
2652        }
2653
2654        if (gr->lkb_highbast < rq->lkb_rqmode && !modes_compat(gr, rq))
2655                return 1;
2656        return 0;
2657}
2658
2659static void send_bast_queue(struct dlm_rsb *r, struct list_head *head,
2660                            struct dlm_lkb *lkb)
2661{
2662        struct dlm_lkb *gr;
2663
2664        list_for_each_entry(gr, head, lkb_statequeue) {
2665                /* skip self when sending basts to convertqueue */
2666                if (gr == lkb)
2667                        continue;
2668                if (gr->lkb_bastfn && modes_require_bast(gr, lkb)) {
2669                        queue_bast(r, gr, lkb->lkb_rqmode);
2670                        gr->lkb_highbast = lkb->lkb_rqmode;
2671                }
2672        }
2673}
2674
2675static void send_blocking_asts(struct dlm_rsb *r, struct dlm_lkb *lkb)
2676{
2677        send_bast_queue(r, &r->res_grantqueue, lkb);
2678}
2679
2680static void send_blocking_asts_all(struct dlm_rsb *r, struct dlm_lkb *lkb)
2681{
2682        send_bast_queue(r, &r->res_grantqueue, lkb);
2683        send_bast_queue(r, &r->res_convertqueue, lkb);
2684}
2685
2686/* set_master(r, lkb) -- set the master nodeid of a resource
2687
2688   The purpose of this function is to set the nodeid field in the given
2689   lkb using the nodeid field in the given rsb.  If the rsb's nodeid is
2690   known, it can just be copied to the lkb and the function will return
2691   0.  If the rsb's nodeid is _not_ known, it needs to be looked up
2692   before it can be copied to the lkb.
2693
2694   When the rsb nodeid is being looked up remotely, the initial lkb
2695   causing the lookup is kept on the ls_waiters list waiting for the
2696   lookup reply.  Other lkb's waiting for the same rsb lookup are kept
2697   on the rsb's res_lookup list until the master is verified.
2698
2699   Return values:
2700   0: nodeid is set in rsb/lkb and the caller should go ahead and use it
2701   1: the rsb master is not available and the lkb has been placed on
2702      a wait queue
2703*/
2704
2705static int set_master(struct dlm_rsb *r, struct dlm_lkb *lkb)
2706{
2707        int our_nodeid = dlm_our_nodeid();
2708
2709        if (rsb_flag(r, RSB_MASTER_UNCERTAIN)) {
2710                rsb_clear_flag(r, RSB_MASTER_UNCERTAIN);
2711                r->res_first_lkid = lkb->lkb_id;
2712                lkb->lkb_nodeid = r->res_nodeid;
2713                return 0;
2714        }
2715
2716        if (r->res_first_lkid && r->res_first_lkid != lkb->lkb_id) {
2717                list_add_tail(&lkb->lkb_rsb_lookup, &r->res_lookup);
2718                return 1;
2719        }
2720
2721        if (r->res_master_nodeid == our_nodeid) {
2722                lkb->lkb_nodeid = 0;
2723                return 0;
2724        }
2725
2726        if (r->res_master_nodeid) {
2727                lkb->lkb_nodeid = r->res_master_nodeid;
2728                return 0;
2729        }
2730
2731        if (dlm_dir_nodeid(r) == our_nodeid) {
2732                /* This is a somewhat unusual case; find_rsb will usually
2733                   have set res_master_nodeid when dir nodeid is local, but
2734                   there are cases where we become the dir node after we've
2735                   past find_rsb and go through _request_lock again.
2736                   confirm_master() or process_lookup_list() needs to be
2737                   called after this. */
2738                log_debug(r->res_ls, "set_master %x self master %d dir %d %s",
2739                          lkb->lkb_id, r->res_master_nodeid, r->res_dir_nodeid,
2740                          r->res_name);
2741                r->res_master_nodeid = our_nodeid;
2742                r->res_nodeid = 0;
2743                lkb->lkb_nodeid = 0;
2744                return 0;
2745        }
2746
2747        wait_pending_remove(r);
2748
2749        r->res_first_lkid = lkb->lkb_id;
2750        send_lookup(r, lkb);
2751        return 1;
2752}
2753
2754static void process_lookup_list(struct dlm_rsb *r)
2755{
2756        struct dlm_lkb *lkb, *safe;
2757
2758        list_for_each_entry_safe(lkb, safe, &r->res_lookup, lkb_rsb_lookup) {
2759                list_del_init(&lkb->lkb_rsb_lookup);
2760                _request_lock(r, lkb);
2761                schedule();
2762        }
2763}
2764
2765/* confirm_master -- confirm (or deny) an rsb's master nodeid */
2766
2767static void confirm_master(struct dlm_rsb *r, int error)
2768{
2769        struct dlm_lkb *lkb;
2770
2771        if (!r->res_first_lkid)
2772                return;
2773
2774        switch (error) {
2775        case 0:
2776        case -EINPROGRESS:
2777                r->res_first_lkid = 0;
2778                process_lookup_list(r);
2779                break;
2780
2781        case -EAGAIN:
2782        case -EBADR:
2783        case -ENOTBLK:
2784                /* the remote request failed and won't be retried (it was
2785                   a NOQUEUE, or has been canceled/unlocked); make a waiting
2786                   lkb the first_lkid */
2787
2788                r->res_first_lkid = 0;
2789
2790                if (!list_empty(&r->res_lookup)) {
2791                        lkb = list_entry(r->res_lookup.next, struct dlm_lkb,
2792                                         lkb_rsb_lookup);
2793                        list_del_init(&lkb->lkb_rsb_lookup);
2794                        r->res_first_lkid = lkb->lkb_id;
2795                        _request_lock(r, lkb);
2796                }
2797                break;
2798
2799        default:
2800                log_error(r->res_ls, "confirm_master unknown error %d", error);
2801        }
2802}
2803
2804static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags,
2805                         int namelen, unsigned long timeout_cs,
2806                         void (*ast) (void *astparam),
2807                         void *astparam,
2808                         void (*bast) (void *astparam, int mode),
2809                         struct dlm_args *args)
2810{
2811        int rv = -EINVAL;
2812
2813        /* check for invalid arg usage */
2814
2815        if (mode < 0 || mode > DLM_LOCK_EX)
2816                goto out;
2817
2818        if (!(flags & DLM_LKF_CONVERT) && (namelen > DLM_RESNAME_MAXLEN))
2819                goto out;
2820
2821        if (flags & DLM_LKF_CANCEL)
2822                goto out;
2823
2824        if (flags & DLM_LKF_QUECVT && !(flags & DLM_LKF_CONVERT))
2825                goto out;
2826
2827        if (flags & DLM_LKF_CONVDEADLK && !(flags & DLM_LKF_CONVERT))
2828                goto out;
2829
2830        if (flags & DLM_LKF_CONVDEADLK && flags & DLM_LKF_NOQUEUE)
2831                goto out;
2832
2833        if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_CONVERT)
2834                goto out;
2835
2836        if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_QUECVT)
2837                goto out;
2838
2839        if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_NOQUEUE)
2840                goto out;
2841
2842        if (flags & DLM_LKF_EXPEDITE && mode != DLM_LOCK_NL)
2843                goto out;
2844
2845        if (!ast || !lksb)
2846                goto out;
2847
2848        if (flags & DLM_LKF_VALBLK && !lksb->sb_lvbptr)
2849                goto out;
2850
2851        if (flags & DLM_LKF_CONVERT && !lksb->sb_lkid)
2852                goto out;
2853
2854        /* these args will be copied to the lkb in validate_lock_args,
2855           it cannot be done now because when converting locks, fields in
2856           an active lkb cannot be modified before locking the rsb */
2857
2858        args->flags = flags;
2859        args->astfn = ast;
2860        args->astparam = astparam;
2861        args->bastfn = bast;
2862        args->timeout = timeout_cs;
2863        args->mode = mode;
2864        args->lksb = lksb;
2865        rv = 0;
2866 out:
2867        return rv;
2868}
2869
2870static int set_unlock_args(uint32_t flags, void *astarg, struct dlm_args *args)
2871{
2872        if (flags & ~(DLM_LKF_CANCEL | DLM_LKF_VALBLK | DLM_LKF_IVVALBLK |
2873                      DLM_LKF_FORCEUNLOCK))
2874                return -EINVAL;
2875
2876        if (flags & DLM_LKF_CANCEL && flags & DLM_LKF_FORCEUNLOCK)
2877                return -EINVAL;
2878
2879        args->flags = flags;
2880        args->astparam = astarg;
2881        return 0;
2882}
2883
2884static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
2885                              struct dlm_args *args)
2886{
2887        int rv = -EINVAL;
2888
2889        if (args->flags & DLM_LKF_CONVERT) {
2890                if (lkb->lkb_flags & DLM_IFL_MSTCPY)
2891                        goto out;
2892
2893                if (args->flags & DLM_LKF_QUECVT &&
2894                    !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1])
2895                        goto out;
2896
2897                rv = -EBUSY;
2898                if (lkb->lkb_status != DLM_LKSTS_GRANTED)
2899                        goto out;
2900
2901                if (lkb->lkb_wait_type)
2902                        goto out;
2903
2904                if (is_overlap(lkb))
2905                        goto out;
2906        }
2907
2908        lkb->lkb_exflags = args->flags;
2909        lkb->lkb_sbflags = 0;
2910        lkb->lkb_astfn = args->astfn;
2911        lkb->lkb_astparam = args->astparam;
2912        lkb->lkb_bastfn = args->bastfn;
2913        lkb->lkb_rqmode = args->mode;
2914        lkb->lkb_lksb = args->lksb;
2915        lkb->lkb_lvbptr = args->lksb->sb_lvbptr;
2916        lkb->lkb_ownpid = (int) current->pid;
2917        lkb->lkb_timeout_cs = args->timeout;
2918        rv = 0;
2919 out:
2920        if (rv)
2921                log_debug(ls, "validate_lock_args %d %x %x %x %d %d %s",
2922                          rv, lkb->lkb_id, lkb->lkb_flags, args->flags,
2923                          lkb->lkb_status, lkb->lkb_wait_type,
2924                          lkb->lkb_resource->res_name);
2925        return rv;
2926}
2927
2928/* when dlm_unlock() sees -EBUSY with CANCEL/FORCEUNLOCK it returns 0
2929   for success */
2930
2931/* note: it's valid for lkb_nodeid/res_nodeid to be -1 when we get here
2932   because there may be a lookup in progress and it's valid to do
2933   cancel/unlockf on it */
2934
2935static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args)
2936{
2937        struct dlm_ls *ls = lkb->lkb_resource->res_ls;
2938        int rv = -EINVAL;
2939
2940        if (lkb->lkb_flags & DLM_IFL_MSTCPY) {
2941                log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id);
2942                dlm_print_lkb(lkb);
2943                goto out;
2944        }
2945
2946        /* an lkb may still exist even though the lock is EOL'ed due to a
2947           cancel, unlock or failed noqueue request; an app can't use these
2948           locks; return same error as if the lkid had not been found at all */
2949
2950        if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) {
2951                log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id);
2952                rv = -ENOENT;
2953                goto out;
2954        }
2955
2956        /* an lkb may be waiting for an rsb lookup to complete where the
2957           lookup was initiated by another lock */
2958
2959        if (!list_empty(&lkb->lkb_rsb_lookup)) {
2960                if (args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) {
2961                        log_debug(ls, "unlock on rsb_lookup %x", lkb->lkb_id);
2962                        list_del_init(&lkb->lkb_rsb_lookup);
2963                        queue_cast(lkb->lkb_resource, lkb,
2964                                   args->flags & DLM_LKF_CANCEL ?
2965                                   -DLM_ECANCEL : -DLM_EUNLOCK);
2966                        unhold_lkb(lkb); /* undoes create_lkb() */
2967                }
2968                /* caller changes -EBUSY to 0 for CANCEL and FORCEUNLOCK */
2969                rv = -EBUSY;
2970                goto out;
2971        }
2972
2973        /* cancel not allowed with another cancel/unlock in progress */
2974
2975        if (args->flags & DLM_LKF_CANCEL) {
2976                if (lkb->lkb_exflags & DLM_LKF_CANCEL)
2977                        goto out;
2978
2979                if (is_overlap(lkb))
2980                        goto out;
2981
2982                /* don't let scand try to do a cancel */
2983                del_timeout(lkb);
2984
2985                if (lkb->lkb_flags & DLM_IFL_RESEND) {
2986                        lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL;
2987                        rv = -EBUSY;
2988                        goto out;
2989                }
2990
2991                /* there's nothing to cancel */
2992                if (lkb->lkb_status == DLM_LKSTS_GRANTED &&
2993                    !lkb->lkb_wait_type) {
2994                        rv = -EBUSY;
2995                        goto out;
2996                }
2997
2998                switch (lkb->lkb_wait_type) {
2999                case DLM_MSG_LOOKUP:
3000                case DLM_MSG_REQUEST:
3001                        lkb->lkb_flags |= DLM_IFL_OVERLAP_CANCEL;
3002                        rv = -EBUSY;
3003                        goto out;
3004                case DLM_MSG_UNLOCK:
3005                case DLM_MSG_CANCEL:
3006                        goto out;
3007                }
3008                /* add_to_waiters() will set OVERLAP_CANCEL */
3009                goto out_ok;
3010        }
3011
3012        /* do we need to allow a force-unlock if there's a normal unlock
3013           already in progress?  in what conditions could the normal unlock
3014           fail such that we'd want to send a force-unlock to be sure? */
3015
3016        if (args->flags & DLM_LKF_FORCEUNLOCK) {
3017                if (lkb->lkb_exflags & DLM_LKF_FORCEUNLOCK)
3018                        goto out;
3019
3020                if (is_overlap_unlock(lkb))
3021                        goto out;
3022
3023                /* don't let scand try to do a cancel */
3024                del_timeout(lkb);
3025
3026                if (lkb->lkb_flags & DLM_IFL_RESEND) {
3027                        lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK;
3028                        rv = -EBUSY;
3029                        goto out;
3030                }
3031
3032                switch (lkb->lkb_wait_type) {
3033                case DLM_MSG_LOOKUP:
3034                case DLM_MSG_REQUEST:
3035                        lkb->lkb_flags |= DLM_IFL_OVERLAP_UNLOCK;
3036                        rv = -EBUSY;
3037                        goto out;
3038                case DLM_MSG_UNLOCK:
3039                        goto out;
3040                }
3041                /* add_to_waiters() will set OVERLAP_UNLOCK */
3042                goto out_ok;
3043        }
3044
3045        /* normal unlock not allowed if there's any op in progress */
3046        rv = -EBUSY;
3047        if (lkb->lkb_wait_type || lkb->lkb_wait_count)
3048                goto out;
3049
3050 out_ok:
3051        /* an overlapping op shouldn't blow away exflags from other op */
3052        lkb->lkb_exflags |= args->flags;
3053        lkb->lkb_sbflags = 0;
3054        lkb->lkb_astparam = args->astparam;
3055        rv = 0;
3056 out:
3057        if (rv)
3058                log_debug(ls, "validate_unlock_args %d %x %x %x %x %d %s", rv,
3059                          lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags,
3060                          args->flags, lkb->lkb_wait_type,
3061                          lkb->lkb_resource->res_name);
3062        return rv;
3063}
3064
3065/*
3066 * Four stage 4 varieties:
3067 * do_request(), do_convert(), do_unlock(), do_cancel()
3068 * These are called on the master node for the given lock and
3069 * from the central locking logic.
3070 */
3071
3072static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
3073{
3074        int error = 0;
3075
3076        if (can_be_granted(r, lkb, 1, 0, NULL)) {
3077                grant_lock(r, lkb);
3078                queue_cast(r, lkb, 0);
3079                goto out;
3080        }
3081
3082        if (can_be_queued(lkb)) {
3083                error = -EINPROGRESS;
3084                add_lkb(r, lkb, DLM_LKSTS_WAITING);
3085                add_timeout(lkb);
3086                goto out;
3087        }
3088
3089        error = -EAGAIN;
3090        queue_cast(r, lkb, -EAGAIN);
3091 out:
3092        return error;
3093}
3094
3095static void do_request_effects(struct dlm_rsb *r, struct dlm_lkb *lkb,
3096                               int error)
3097{
3098        switch (error) {
3099        case -EAGAIN:
3100                if (force_blocking_asts(lkb))
3101                        send_blocking_asts_all(r, lkb);
3102                break;
3103        case -EINPROGRESS:
3104                send_blocking_asts(r, lkb);
3105                break;
3106        }
3107}
3108
3109static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb)
3110{
3111        int error = 0;
3112        int deadlk = 0;
3113
3114        /* changing an existing lock may allow others to be granted */
3115
3116        if (can_be_granted(r, lkb, 1, 0, &deadlk)) {
3117                grant_lock(r, lkb);
3118                queue_cast(r, lkb, 0);
3119                goto out;
3120        }
3121
3122        /* can_be_granted() detected that this lock would block in a conversion
3123           deadlock, so we leave it on the granted queue and return EDEADLK in
3124           the ast for the convert. */
3125
3126        if (deadlk) {
3127                /* it's left on the granted queue */
3128                revert_lock(r, lkb);
3129                queue_cast(r, lkb, -EDEADLK);
3130                error = -EDEADLK;
3131                goto out;
3132        }
3133
3134        /* is_demoted() means the can_be_granted() above set the grmode
3135           to NL, and left us on the granted queue.  This auto-demotion
3136           (due to CONVDEADLK) might mean other locks, and/or this lock, are
3137           now grantable.  We have to try to grant other converting locks
3138           before we try again to grant this one. */
3139
3140        if (is_demoted(lkb)) {
3141                grant_pending_convert(r, DLM_LOCK_IV, NULL, NULL);
3142                if (_can_be_granted(r, lkb, 1, 0)) {
3143                        grant_lock(r, lkb);
3144                        queue_cast(r, lkb, 0);
3145                        goto out;
3146                }
3147                /* else fall through and move to convert queue */
3148        }
3149
3150        if (can_be_queued(lkb)) {
3151                error = -EINPROGRESS;
3152                del_lkb(r, lkb);
3153                add_lkb(r, lkb, DLM_LKSTS_CONVERT);
3154                add_timeout(lkb);
3155                goto out;
3156        }
3157
3158        error = -EAGAIN;
3159        queue_cast(r, lkb, -EAGAIN);
3160 out:
3161        return error;
3162}
3163
3164static void do_convert_effects(struct dlm_rsb *r, struct dlm_lkb *lkb,
3165                               int error)
3166{
3167        switch (error) {
3168        case 0:
3169                grant_pending_locks(r, NULL);
3170                /* grant_pending_locks also sends basts */
3171                break;
3172        case -EAGAIN:
3173                if (force_blocking_asts(lkb))
3174                        send_blocking_asts_all(r, lkb);
3175                break;
3176        case -EINPROGRESS:
3177                send_blocking_asts(r, lkb);
3178                break;
3179        }
3180}
3181
3182static int do_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb)
3183{
3184        remove_lock(r, lkb);
3185        queue_cast(r, lkb, -DLM_EUNLOCK);
3186        return -DLM_EUNLOCK;
3187}
3188
3189static void do_unlock_effects(struct dlm_rsb *r, struct dlm_lkb *lkb,
3190                              int error)
3191{
3192        grant_pending_locks(r, NULL);
3193}
3194
3195/* returns: 0 did nothing, -DLM_ECANCEL canceled lock */
3196
3197static int do_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb)
3198{
3199        int error;
3200
3201        error = revert_lock(r, lkb);
3202        if (error) {
3203                queue_cast(r, lkb, -DLM_ECANCEL);
3204                return -DLM_ECANCEL;
3205        }
3206        return 0;
3207}
3208
3209static void do_cancel_effects(struct dlm_rsb *r, struct dlm_lkb *lkb,
3210                              int error)
3211{
3212        if (error)
3213                grant_pending_locks(r, NULL);
3214}
3215
3216/*
3217 * Four stage 3 varieties:
3218 * _request_lock(), _convert_lock(), _unlock_lock(), _cancel_lock()
3219 */
3220
3221/* add a new lkb to a possibly new rsb, called by requesting process */
3222
3223static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
3224{
3225        int error;
3226
3227        /* set_master: sets lkb nodeid from r */
3228
3229        error = set_master(r, lkb);
3230        if (error < 0)
3231                goto out;
3232        if (error) {
3233                error = 0;
3234                goto out;
3235        }
3236
3237        if (is_remote(r)) {
3238                /* receive_request() calls do_request() on remote node */
3239                error = send_request(r, lkb);
3240        } else {
3241                error = do_request(r, lkb);
3242                /* for remote locks the request_reply is sent
3243                   between do_request and do_request_effects */
3244                do_request_effects(r, lkb, error);
3245        }
3246 out:
3247        return error;
3248}
3249
3250/* change some property of an existing lkb, e.g. mode */
3251
3252static int _convert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
3253{
3254        int error;
3255
3256        if (is_remote(r)) {
3257                /* receive_convert() calls do_convert() on remote node */
3258                error = send_convert(r, lkb);
3259        } else {
3260                error = do_convert(r, lkb);
3261                /* for remote locks the convert_reply is sent
3262                   between do_convert and do_convert_effects */
3263                do_convert_effects(r, lkb, error);
3264        }
3265
3266        return error;
3267}
3268
3269/* remove an existing lkb from the granted queue */
3270
3271static int _unlock_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
3272{
3273        int error;
3274
3275        if (is_remote(r)) {
3276                /* receive_unlock() calls do_unlock() on remote node */
3277                error = send_unlock(r, lkb);
3278        } else {
3279                error = do_unlock(r, lkb);
3280                /* for remote locks the unlock_reply is sent
3281                   between do_unlock and do_unlock_effects */
3282                do_unlock_effects(r, lkb, error);
3283        }
3284
3285        return error;
3286}
3287
3288/* remove an existing lkb from the convert or wait queue */
3289
3290static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
3291{
3292        int error;
3293
3294        if (is_remote(r)) {
3295                /* receive_cancel() calls do_cancel() on remote node */
3296                error = send_cancel(r, lkb);
3297        } else {
3298                error = do_cancel(r, lkb);
3299                /* for remote locks the cancel_reply is sent
3300                   between do_cancel and do_cancel_effects */
3301                do_cancel_effects(r, lkb, error);
3302        }
3303
3304        return error;
3305}
3306
3307/*
3308 * Four stage 2 varieties:
3309 * request_lock(), convert_lock(), unlock_lock(), cancel_lock()
3310 */
3311
3312static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, char *name,
3313                        int len, struct dlm_args *args)
3314{
3315        struct dlm_rsb *r;
3316        int error;
3317
3318        error = validate_lock_args(ls, lkb, args);
3319        if (error)
3320                return error;
3321
3322        error = find_rsb(ls, name, len, 0, R_REQUEST, &r);
3323        if (error)
3324                return error;
3325
3326        lock_rsb(r);
3327
3328        attach_lkb(r, lkb);
3329        lkb->lkb_lksb->sb_lkid = lkb->lkb_id;
3330
3331        error = _request_lock(r, lkb);
3332
3333        unlock_rsb(r);
3334        put_rsb(r);
3335        return error;
3336}
3337
3338static int convert_lock(struct dlm_ls *ls, struct dlm_lkb *lkb,
3339                        struct dlm_args *args)
3340{
3341        struct dlm_rsb *r;
3342        int error;
3343
3344        r = lkb->lkb_resource;
3345
3346        hold_rsb(r);
3347        lock_rsb(r);
3348
3349        error = validate_lock_args(ls, lkb, args);
3350        if (error)
3351                goto out;
3352
3353        error = _convert_lock(r, lkb);
3354 out:
3355        unlock_rsb(r);
3356        put_rsb(r);
3357        return error;
3358}
3359
3360static int unlock_lock(struct dlm_ls *ls, struct dlm_lkb *lkb,
3361                       struct dlm_args *args)
3362{
3363        struct dlm_rsb *r;
3364        int error;
3365
3366        r = lkb->lkb_resource;
3367
3368        hold_rsb(r);
3369        lock_rsb(r);
3370
3371        error = validate_unlock_args(lkb, args);
3372        if (error)
3373                goto out;
3374
3375        error = _unlock_lock(r, lkb);
3376 out:
3377        unlock_rsb(r);
3378        put_rsb(r);
3379        return error;
3380}
3381
3382static int cancel_lock(struct dlm_ls *ls, struct dlm_lkb *lkb,
3383                       struct dlm_args *args)
3384{
3385        struct dlm_rsb *r;
3386        int error;
3387
3388        r = lkb->lkb_resource;
3389
3390        hold_rsb(r);
3391        lock_rsb(r);
3392
3393        error = validate_unlock_args(lkb, args);
3394        if (error)
3395                goto out;
3396
3397        error = _cancel_lock(r, lkb);
3398 out:
3399        unlock_rsb(r);
3400        put_rsb(r);
3401        return error;
3402}
3403
3404/*
3405 * Two stage 1 varieties:  dlm_lock() and dlm_unlock()
3406 */
3407
3408int dlm_lock(dlm_lockspace_t *lockspace,
3409             int mode,
3410             struct dlm_lksb *lksb,
3411             uint32_t flags,
3412             void *name,
3413             unsigned int namelen,
3414             uint32_t parent_lkid,
3415             void (*ast) (void *astarg),
3416             void *astarg,
3417             void (*bast) (void *astarg, int mode))
3418{
3419        struct dlm_ls *ls;
3420        struct dlm_lkb *lkb;
3421        struct dlm_args args;
3422        int error, convert = flags & DLM_LKF_CONVERT;
3423
3424        ls = dlm_find_lockspace_local(lockspace);
3425        if (!ls)
3426                return -EINVAL;
3427
3428        dlm_lock_recovery(ls);
3429
3430        if (convert)
3431                error = find_lkb(ls, lksb->sb_lkid, &lkb);
3432        else
3433                error = create_lkb(ls, &lkb);
3434
3435        if (error)
3436                goto out;
3437
3438        error = set_lock_args(mode, lksb, flags, namelen, 0, ast,
3439                              astarg, bast, &args);
3440        if (error)
3441                goto out_put;
3442
3443        if (convert)
3444                error = convert_lock(ls, lkb, &args);
3445        else
3446                error = request_lock(ls, lkb, name, namelen, &args);
3447
3448        if (error == -EINPROGRESS)
3449                error = 0;
3450 out_put:
3451        if (convert || error)
3452                __put_lkb(ls, lkb);
3453        if (error == -EAGAIN || error == -EDEADLK)
3454                error = 0;
3455 out:
3456        dlm_unlock_recovery(ls);
3457        dlm_put_lockspace(ls);
3458        return error;
3459}
3460
3461int dlm_unlock(dlm_lockspace_t *lockspace,
3462               uint32_t lkid,
3463               uint32_t flags,
3464               struct dlm_lksb *lksb,
3465               void *astarg)
3466{
3467        struct dlm_ls *ls;
3468        struct dlm_lkb *lkb;
3469        struct dlm_args args;
3470        int error;
3471
3472        ls = dlm_find_lockspace_local(lockspace);
3473        if (!ls)
3474                return -EINVAL;
3475
3476        dlm_lock_recovery(ls);
3477
3478        error = find_lkb(ls, lkid, &lkb);
3479        if (error)
3480                goto out;
3481
3482        error = set_unlock_args(flags, astarg, &args);
3483        if (error)
3484                goto out_put;
3485
3486        if (flags & DLM_LKF_CANCEL)
3487                error = cancel_lock(ls, lkb, &args);
3488        else
3489                error = unlock_lock(ls, lkb, &args);
3490
3491        if (error == -DLM_EUNLOCK || error == -DLM_ECANCEL)
3492                error = 0;
3493        if (error == -EBUSY && (flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)))
3494                error = 0;
3495 out_put:
3496        dlm_put_lkb(lkb);
3497 out:
3498        dlm_unlock_recovery(ls);
3499        dlm_put_lockspace(ls);
3500        return error;
3501}
3502
3503/*
3504 * send/receive routines for remote operations and replies
3505 *
3506 * send_args
3507 * send_common
3508 * send_request                 receive_request
3509 * send_convert                 receive_convert
3510 * send_unlock                  receive_unlock
3511 * send_cancel                  receive_cancel
3512 * send_grant                   receive_grant
3513 * send_bast                    receive_bast
3514 * send_lookup                  receive_lookup
3515 * send_remove                  receive_remove
3516 *
3517 *                              send_common_reply
3518 * receive_request_reply        send_request_reply
3519 * receive_convert_reply        send_convert_reply
3520 * receive_unlock_reply         send_unlock_reply
3521 * receive_cancel_reply         send_cancel_reply
3522 * receive_lookup_reply         send_lookup_reply
3523 */
3524
3525static int _create_message(struct dlm_ls *ls, int mb_len,
3526                           int to_nodeid, int mstype,
3527                           struct dlm_message **ms_ret,
3528                           struct dlm_mhandle **mh_ret)
3529{
3530        struct dlm_message *ms;
3531        struct dlm_mhandle *mh;
3532        char *mb;
3533
3534        /* get_buffer gives us a message handle (mh) that we need to
3535           pass into lowcomms_commit and a message buffer (mb) that we
3536           write our data into */
3537
3538        mh = dlm_lowcomms_get_buffer(to_nodeid, mb_len, GFP_NOFS, &mb);
3539        if (!mh)
3540                return -ENOBUFS;
3541
3542        memset(mb, 0, mb_len);
3543
3544        ms = (struct dlm_message *) mb;
3545
3546        ms->m_header.h_version = (DLM_HEADER_MAJOR | DLM_HEADER_MINOR);
3547        ms->m_header.h_lockspace = ls->ls_global_id;
3548        ms->m_header.h_nodeid = dlm_our_nodeid();
3549        ms->m_header.h_length = mb_len;
3550        ms->m_header.h_cmd = DLM_MSG;
3551
3552        ms->m_type = mstype;
3553
3554        *mh_ret = mh;
3555        *ms_ret = ms;
3556        return 0;
3557}
3558
3559static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb,
3560                          int to_nodeid, int mstype,
3561                          struct dlm_message **ms_ret,
3562                          struct dlm_mhandle **mh_ret)
3563{
3564        int mb_len = sizeof(struct dlm_message);
3565
3566        switch (mstype) {
3567        case DLM_MSG_REQUEST:
3568        case DLM_MSG_LOOKUP:
3569        case DLM_MSG_REMOVE:
3570                mb_len += r->res_length;
3571                break;
3572        case DLM_MSG_CONVERT:
3573        case DLM_MSG_UNLOCK:
3574        case DLM_MSG_REQUEST_REPLY:
3575        case DLM_MSG_CONVERT_REPLY:
3576        case DLM_MSG_GRANT:
3577                if (lkb && lkb->lkb_lvbptr)
3578                        mb_len += r->res_ls->ls_lvblen;
3579                break;
3580        }
3581
3582        return _create_message(r->res_ls, mb_len, to_nodeid, mstype,
3583                               ms_ret, mh_ret);
3584}
3585
3586/* further lowcomms enhancements or alternate implementations may make
3587   the return value from this function useful at some point */
3588
3589static int send_message(struct dlm_mhandle *mh, struct dlm_message *ms)
3590{
3591        dlm_message_out(ms);
3592        dlm_lowcomms_commit_buffer(mh);
3593        return 0;
3594}
3595
3596static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb,
3597                      struct dlm_message *ms)
3598{
3599        ms->m_nodeid   = lkb->lkb_nodeid;
3600        ms->m_pid      = lkb->lkb_ownpid;
3601        ms->m_lkid     = lkb->lkb_id;
3602        ms->m_remid    = lkb->lkb_remid;
3603        ms->m_exflags  = lkb->lkb_exflags;
3604        ms->m_sbflags  = lkb->lkb_sbflags;
3605        ms->m_flags    = lkb->lkb_flags;
3606        ms->m_lvbseq   = lkb->lkb_lvbseq;
3607        ms->m_status   = lkb->lkb_status;
3608        ms->m_grmode   = lkb->lkb_grmode;
3609        ms->m_rqmode   = lkb->lkb_rqmode;
3610        ms->m_hash     = r->res_hash;
3611
3612        /* m_result and m_bastmode are set from function args,
3613           not from lkb fields */
3614
3615        if (lkb->lkb_bastfn)
3616                ms->m_asts |= DLM_CB_BAST;
3617        if (lkb->lkb_astfn)
3618                ms->m_asts |= DLM_CB_CAST;
3619
3620        /* compare with switch in create_message; send_remove() doesn't
3621           use send_args() */
3622
3623        switch (ms->m_type) {
3624        case DLM_MSG_REQUEST:
3625        case DLM_MSG_LOOKUP:
3626                memcpy(ms->m_extra, r->res_name, r->res_length);
3627                break;
3628        case DLM_MSG_CONVERT:
3629        case DLM_MSG_UNLOCK:
3630        case DLM_MSG_REQUEST_REPLY:
3631        case DLM_MSG_CONVERT_REPLY:
3632        case DLM_MSG_GRANT:
3633                if (!lkb->lkb_lvbptr)
3634                        break;
3635                memcpy(ms->m_extra, lkb->lkb_lvbptr, r->res_ls->ls_lvblen);
3636                break;
3637        }
3638}
3639
3640static int send_common(struct dlm_rsb *r, struct dlm_lkb *lkb, int mstype)
3641{
3642        struct dlm_message *ms;
3643        struct dlm_mhandle *mh;
3644        int to_nodeid, error;
3645
3646        to_nodeid = r->res_nodeid;
3647
3648        error = add_to_waiters(lkb, mstype, to_nodeid);
3649        if (error)
3650                return error;
3651
3652        error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh);
3653        if (error)
3654                goto fail;
3655
3656        send_args(r, lkb, ms);
3657
3658        error = send_message(mh, ms);
3659        if (error)
3660                goto fail;
3661        return 0;
3662
3663 fail:
3664        remove_from_waiters(lkb, msg_reply_type(mstype));
3665        return error;
3666}
3667
3668static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
3669{
3670        return send_common(r, lkb, DLM_MSG_REQUEST);
3671}
3672
3673static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb)
3674{
3675        int error;
3676
3677        error = send_common(r, lkb, DLM_MSG_CONVERT);
3678
3679        /* down conversions go without a reply from the master */
3680        if (!error && down_conversion(lkb)) {
3681                remove_from_waiters(lkb, DLM_MSG_CONVERT_REPLY);
3682                r->res_ls->ls_stub_ms.m_flags = DLM_IFL_STUB_MS;
3683                r->res_ls->ls_stub_ms.m_type = DLM_MSG_CONVERT_REPLY;
3684                r->res_ls->ls_stub_ms.m_result = 0;
3685                __receive_convert_reply(r, lkb, &r->res_ls->ls_stub_ms);
3686        }
3687
3688        return error;
3689}
3690
3691/* FIXME: if this lkb is the only lock we hold on the rsb, then set
3692   MASTER_UNCERTAIN to force the next request on the rsb to confirm
3693   that the master is still correct. */
3694
3695static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb)
3696{
3697        return send_common(r, lkb, DLM_MSG_UNLOCK);
3698}
3699
3700static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb)
3701{
3702        return send_common(r, lkb, DLM_MSG_CANCEL);
3703}
3704
3705static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb)
3706{
3707        struct dlm_message *ms;
3708        struct dlm_mhandle *mh;
3709        int to_nodeid, error;
3710
3711        to_nodeid = lkb->lkb_nodeid;
3712
3713        error = create_message(r, lkb, to_nodeid, DLM_MSG_GRANT, &ms, &mh);
3714        if (error)
3715                goto out;
3716
3717        send_args(r, lkb, ms);
3718
3719        ms->m_result = 0;
3720
3721        error = send_message(mh, ms);
3722 out:
3723        return error;
3724}
3725
3726static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode)
3727{
3728        struct dlm_message *ms;
3729        struct dlm_mhandle *mh;
3730        int to_nodeid, error;
3731
3732        to_nodeid = lkb->lkb_nodeid;
3733
3734        error = create_message(r, NULL, to_nodeid, DLM_MSG_BAST, &ms, &mh);
3735        if (error)
3736                goto out;
3737
3738        send_args(r, lkb, ms);
3739
3740        ms->m_bastmode = mode;
3741
3742        error = send_message(mh, ms);
3743 out:
3744        return error;
3745}
3746
3747static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb)
3748{
3749        struct dlm_message *ms;
3750        struct dlm_mhandle *mh;
3751        int to_nodeid, error;
3752
3753        to_nodeid = dlm_dir_nodeid(r);
3754
3755        error = add_to_waiters(lkb, DLM_MSG_LOOKUP, to_nodeid);
3756        if (error)
3757                return error;
3758
3759        error = create_message(r, NULL, to_nodeid, DLM_MSG_LOOKUP, &ms, &mh);
3760        if (error)
3761                goto fail;
3762
3763        send_args(r, lkb, ms);
3764
3765        error = send_message(mh, ms);
3766        if (error)
3767                goto fail;
3768        return 0;
3769
3770 fail:
3771        remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY);
3772        return error;
3773}
3774
3775static int send_remove(struct dlm_rsb *r)
3776{
3777        struct dlm_message *ms;
3778        struct dlm_mhandle *mh;
3779        int to_nodeid, error;
3780
3781        to_nodeid = dlm_dir_nodeid(r);
3782
3783        error = create_message(r, NULL, to_nodeid, DLM_MSG_REMOVE, &ms, &mh);
3784        if (error)
3785                goto out;
3786
3787        memcpy(ms->m_extra, r->res_name, r->res_length);
3788        ms->m_hash = r->res_hash;
3789
3790        error = send_message(mh, ms);
3791 out:
3792        return error;
3793}
3794
3795static int send_common_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
3796                             int mstype, int rv)
3797{
3798        struct dlm_message *ms;
3799        struct dlm_mhandle *mh;
3800        int to_nodeid, error;
3801
3802        to_nodeid = lkb->lkb_nodeid;
3803
3804        error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh);
3805        if (error)
3806                goto out;
3807
3808        send_args(r, lkb, ms);
3809
3810        ms->m_result = rv;
3811
3812        error = send_message(mh, ms);
3813 out:
3814        return error;
3815}
3816
3817static int send_request_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
3818{
3819        return send_common_reply(r, lkb, DLM_MSG_REQUEST_REPLY, rv);
3820}
3821
3822static int send_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
3823{
3824        return send_common_reply(r, lkb, DLM_MSG_CONVERT_REPLY, rv);
3825}
3826
3827static int send_unlock_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
3828{
3829        return send_common_reply(r, lkb, DLM_MSG_UNLOCK_REPLY, rv);
3830}
3831
3832static int send_cancel_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv)
3833{
3834        return send_common_reply(r, lkb, DLM_MSG_CANCEL_REPLY, rv);
3835}
3836
3837static int send_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms_in,
3838                             int ret_nodeid, int rv)
3839{
3840        struct dlm_rsb *r = &ls->ls_stub_rsb;
3841        struct dlm_message *ms;
3842        struct dlm_mhandle *mh;
3843        int error, nodeid = ms_in->m_header.h_nodeid;
3844
3845        error = create_message(r, NULL, nodeid, DLM_MSG_LOOKUP_REPLY, &ms, &mh);
3846        if (error)
3847                goto out;
3848
3849        ms->m_lkid = ms_in->m_lkid;
3850        ms->m_result = rv;
3851        ms->m_nodeid = ret_nodeid;
3852
3853        error = send_message(mh, ms);
3854 out:
3855        return error;
3856}
3857
3858/* which args we save from a received message depends heavily on the type
3859   of message, unlike the send side where we can safely send everything about
3860   the lkb for any type of message */
3861
3862static void receive_flags(struct dlm_lkb *lkb, struct dlm_message *ms)
3863{
3864        lkb->lkb_exflags = ms->m_exflags;
3865        lkb->lkb_sbflags = ms->m_sbflags;
3866        lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) |
3867                         (ms->m_flags & 0x0000FFFF);
3868}
3869
3870static void receive_flags_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
3871{
3872        if (ms->m_flags == DLM_IFL_STUB_MS)
3873                return;
3874
3875        lkb->lkb_sbflags = ms->m_sbflags;
3876        lkb->lkb_flags = (lkb->lkb_flags & 0xFFFF0000) |
3877                         (ms->m_flags & 0x0000FFFF);
3878}
3879
3880static int receive_extralen(struct dlm_message *ms)
3881{
3882        return (ms->m_header.h_length - sizeof(struct dlm_message));
3883}
3884
3885static int receive_lvb(struct dlm_ls *ls, struct dlm_lkb *lkb,
3886                       struct dlm_message *ms)
3887{
3888        int len;
3889
3890        if (lkb->lkb_exflags & DLM_LKF_VALBLK) {
3891                if (!lkb->lkb_lvbptr)
3892                        lkb->lkb_lvbptr = dlm_allocate_lvb(ls);
3893                if (!lkb->lkb_lvbptr)
3894                        return -ENOMEM;
3895                len = receive_extralen(ms);
3896                if (len > ls->ls_lvblen)
3897                        len = ls->ls_lvblen;
3898                memcpy(lkb->lkb_lvbptr, ms->m_extra, len);
3899        }
3900        return 0;
3901}
3902
3903static void fake_bastfn(void *astparam, int mode)
3904{
3905        log_print("fake_bastfn should not be called");
3906}
3907
3908static void fake_astfn(void *astparam)
3909{
3910        log_print("fake_astfn should not be called");
3911}
3912
3913static int receive_request_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
3914                                struct dlm_message *ms)
3915{
3916        lkb->lkb_nodeid = ms->m_header.h_nodeid;
3917        lkb->lkb_ownpid = ms->m_pid;
3918        lkb->lkb_remid = ms->m_lkid;
3919        lkb->lkb_grmode = DLM_LOCK_IV;
3920        lkb->lkb_rqmode = ms->m_rqmode;
3921
3922        lkb->lkb_bastfn = (ms->m_asts & DLM_CB_BAST) ? &fake_bastfn : NULL;
3923        lkb->lkb_astfn = (ms->m_asts & DLM_CB_CAST) ? &fake_astfn : NULL;
3924
3925        if (lkb->lkb_exflags & DLM_LKF_VALBLK) {
3926                /* lkb was just created so there won't be an lvb yet */
3927                lkb->lkb_lvbptr = dlm_allocate_lvb(ls);
3928                if (!lkb->lkb_lvbptr)
3929                        return -ENOMEM;
3930        }
3931
3932        return 0;
3933}
3934
3935static int receive_convert_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
3936                                struct dlm_message *ms)
3937{
3938        if (lkb->lkb_status != DLM_LKSTS_GRANTED)
3939                return -EBUSY;
3940
3941        if (receive_lvb(ls, lkb, ms))
3942                return -ENOMEM;
3943
3944        lkb->lkb_rqmode = ms->m_rqmode;
3945        lkb->lkb_lvbseq = ms->m_lvbseq;
3946
3947        return 0;
3948}
3949
3950static int receive_unlock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
3951                               struct dlm_message *ms)
3952{
3953        if (receive_lvb(ls, lkb, ms))
3954                return -ENOMEM;
3955        return 0;
3956}
3957
3958/* We fill in the stub-lkb fields with the info that send_xxxx_reply()
3959   uses to send a reply and that the remote end uses to process the reply. */
3960
3961static void setup_stub_lkb(struct dlm_ls *ls, struct dlm_message *ms)
3962{
3963        struct dlm_lkb *lkb = &ls->ls_stub_lkb;
3964        lkb->lkb_nodeid = ms->m_header.h_nodeid;
3965        lkb->lkb_remid = ms->m_lkid;
3966}
3967
3968/* This is called after the rsb is locked so that we can safely inspect
3969   fields in the lkb. */
3970
3971static int validate_message(struct dlm_lkb *lkb, struct dlm_message *ms)
3972{
3973        int from = ms->m_header.h_nodeid;
3974        int error = 0;
3975
3976        switch (ms->m_type) {
3977        case DLM_MSG_CONVERT:
3978        case DLM_MSG_UNLOCK:
3979        case DLM_MSG_CANCEL:
3980                if (!is_master_copy(lkb) || lkb->lkb_nodeid != from)
3981                        error = -EINVAL;
3982                break;
3983
3984        case DLM_MSG_CONVERT_REPLY:
3985        case DLM_MSG_UNLOCK_REPLY:
3986        case DLM_MSG_CANCEL_REPLY:
3987        case DLM_MSG_GRANT:
3988        case DLM_MSG_BAST:
3989                if (!is_process_copy(lkb) || lkb->lkb_nodeid != from)
3990                        error = -EINVAL;
3991                break;
3992
3993        case DLM_MSG_REQUEST_REPLY:
3994                if (!is_process_copy(lkb))
3995                        error = -EINVAL;
3996                else if (lkb->lkb_nodeid != -1 && lkb->lkb_nodeid != from)
3997                        error = -EINVAL;
3998                break;
3999
4000        default:
4001                error = -EINVAL;
4002        }
4003
4004        if (error)
4005                log_error(lkb->lkb_resource->res_ls,
4006                          "ignore invalid message %d from %d %x %x %x %d",
4007                          ms->m_type, from, lkb->lkb_id, lkb->lkb_remid,
4008                          lkb->lkb_flags, lkb->lkb_nodeid);
4009        return error;
4010}
4011
4012static void send_repeat_remove(struct dlm_ls *ls, char *ms_name, int len)
4013{
4014        char name[DLM_RESNAME_MAXLEN + 1];
4015        struct dlm_message *ms;
4016        struct dlm_mhandle *mh;
4017        struct dlm_rsb *r;
4018        uint32_t hash, b;
4019        int rv, dir_nodeid;
4020
4021        memset(name, 0, sizeof(name));
4022        memcpy(name, ms_name, len);
4023
4024        hash = jhash(name, len, 0);
4025        b = hash & (ls->ls_rsbtbl_size - 1);
4026
4027        dir_nodeid = dlm_hash2nodeid(ls, hash);
4028
4029        log_error(ls, "send_repeat_remove dir %d %s", dir_nodeid, name);
4030
4031        spin_lock(&ls->ls_rsbtbl[b].lock);
4032        rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r);
4033        if (!rv) {
4034                spin_unlock(&ls->ls_rsbtbl[b].lock);
4035                log_error(ls, "repeat_remove on keep %s", name);
4036                return;
4037        }
4038
4039        rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r);
4040        if (!rv) {
4041                spin_unlock(&ls->ls_rsbtbl[b].lock);
4042                log_error(ls, "repeat_remove on toss %s", name);
4043                return;
4044        }
4045
4046        /* use ls->remove_name2 to avoid conflict with shrink? */
4047
4048        spin_lock(&ls->ls_remove_spin);
4049        ls->ls_remove_len = len;
4050        memcpy(ls->ls_remove_name, name, DLM_RESNAME_MAXLEN);
4051        spin_unlock(&ls->ls_remove_spin);
4052        spin_unlock(&ls->ls_rsbtbl[b].lock);
4053
4054        rv = _create_message(ls, sizeof(struct dlm_message) + len,
4055                             dir_nodeid, DLM_MSG_REMOVE, &ms, &mh);
4056        if (rv)
4057                return;
4058
4059        memcpy(ms->m_extra, name, len);
4060        ms->m_hash = hash;
4061
4062        send_message(mh, ms);
4063
4064        spin_lock(&ls->ls_remove_spin);
4065        ls->ls_remove_len = 0;
4066        memset(ls->ls_remove_name, 0, DLM_RESNAME_MAXLEN);
4067        spin_unlock(&ls->ls_remove_spin);
4068}
4069
4070static int receive_request(struct dlm_ls *ls, struct dlm_message *ms)
4071{
4072        struct dlm_lkb *lkb;
4073        struct dlm_rsb *r;
4074        int from_nodeid;
4075        int error, namelen = 0;
4076
4077        from_nodeid = ms->m_header.h_nodeid;
4078
4079        error = create_lkb(ls, &lkb);
4080        if (error)
4081                goto fail;
4082
4083        receive_flags(lkb, ms);
4084        lkb->lkb_flags |= DLM_IFL_MSTCPY;
4085        error = receive_request_args(ls, lkb, ms);
4086        if (error) {
4087                __put_lkb(ls, lkb);
4088                goto fail;
4089        }
4090
4091        /* The dir node is the authority on whether we are the master
4092           for this rsb or not, so if the master sends us a request, we should
4093           recreate the rsb if we've destroyed it.   This race happens when we
4094           send a remove message to the dir node at the same time that the dir
4095           node sends us a request for the rsb. */
4096
4097        namelen = receive_extralen(ms);
4098
4099        error = find_rsb(ls, ms->m_extra, namelen, from_nodeid,
4100                         R_RECEIVE_REQUEST, &r);
4101        if (error) {
4102                __put_lkb(ls, lkb);
4103                goto fail;
4104        }
4105
4106        lock_rsb(r);
4107
4108        if (r->res_master_nodeid != dlm_our_nodeid()) {
4109                error = validate_master_nodeid(ls, r, from_nodeid);
4110                if (error) {
4111                        unlock_rsb(r);
4112                        put_rsb(r);
4113                        __put_lkb(ls, lkb);
4114                        goto fail;
4115                }
4116        }
4117
4118        attach_lkb(r, lkb);
4119        error = do_request(r, lkb);
4120        send_request_reply(r, lkb, error);
4121        do_request_effects(r, lkb, error);
4122
4123        unlock_rsb(r);
4124        put_rsb(r);
4125
4126        if (error == -EINPROGRESS)
4127                error = 0;
4128        if (error)
4129                dlm_put_lkb(lkb);
4130        return 0;
4131
4132 fail:
4133        /* TODO: instead of returning ENOTBLK, add the lkb to res_lookup
4134           and do this receive_request again from process_lookup_list once
4135           we get the lookup reply.  This would avoid a many repeated
4136           ENOTBLK request failures when the lookup reply designating us
4137           as master is delayed. */
4138
4139        /* We could repeatedly return -EBADR here if our send_remove() is
4140           delayed in being sent/arriving/being processed on the dir node.
4141           Another node would repeatedly lookup up the master, and the dir
4142           node would continue returning our nodeid until our send_remove
4143           took effect.
4144
4145           We send another remove message in case our previous send_remove
4146           was lost/ignored/missed somehow. */
4147
4148        if (error != -ENOTBLK) {
4149                log_limit(ls, "receive_request %x from %d %d",
4150                          ms->m_lkid, from_nodeid, error);
4151        }
4152
4153        if (namelen && error == -EBADR) {
4154                send_repeat_remove(ls, ms->m_extra, namelen);
4155                msleep(1000);
4156        }
4157
4158        setup_stub_lkb(ls, ms);
4159        send_request_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
4160        return error;
4161}
4162
4163static int receive_convert(struct dlm_ls *ls, struct dlm_message *ms)
4164{
4165        struct dlm_lkb *lkb;
4166        struct dlm_rsb *r;
4167        int error, reply = 1;
4168
4169        error = find_lkb(ls, ms->m_remid, &lkb);
4170        if (error)
4171                goto fail;
4172
4173        if (lkb->lkb_remid != ms->m_lkid) {
4174                log_error(ls, "receive_convert %x remid %x recover_seq %llu "
4175                          "remote %d %x", lkb->lkb_id, lkb->lkb_remid,
4176                          (unsigned long long)lkb->lkb_recover_seq,
4177                          ms->m_header.h_nodeid, ms->m_lkid);
4178                error = -ENOENT;
4179                goto fail;
4180        }
4181
4182        r = lkb->lkb_resource;
4183
4184        hold_rsb(r);
4185        lock_rsb(r);
4186
4187        error = validate_message(lkb, ms);
4188        if (error)
4189                goto out;
4190
4191        receive_flags(lkb, ms);
4192
4193        error = receive_convert_args(ls, lkb, ms);
4194        if (error) {
4195                send_convert_reply(r, lkb, error);
4196                goto out;
4197        }
4198
4199        reply = !down_conversion(lkb);
4200
4201        error = do_convert(r, lkb);
4202        if (reply)
4203                send_convert_reply(r, lkb, error);
4204        do_convert_effects(r, lkb, error);
4205 out:
4206        unlock_rsb(r);
4207        put_rsb(r);
4208        dlm_put_lkb(lkb);
4209        return 0;
4210
4211 fail:
4212        setup_stub_lkb(ls, ms);
4213        send_convert_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
4214        return error;
4215}
4216
4217static int receive_unlock(struct dlm_ls *ls, struct dlm_message *ms)
4218{
4219        struct dlm_lkb *lkb;
4220        struct dlm_rsb *r;
4221        int error;
4222
4223        error = find_lkb(ls, ms->m_remid, &lkb);
4224        if (error)
4225                goto fail;
4226
4227        if (lkb->lkb_remid != ms->m_lkid) {
4228                log_error(ls, "receive_unlock %x remid %x remote %d %x",
4229                          lkb->lkb_id, lkb->lkb_remid,
4230                          ms->m_header.h_nodeid, ms->m_lkid);
4231                error = -ENOENT;
4232                goto fail;
4233        }
4234
4235        r = lkb->lkb_resource;
4236
4237        hold_rsb(r);
4238        lock_rsb(r);
4239
4240        error = validate_message(lkb, ms);
4241        if (error)
4242                goto out;
4243
4244        receive_flags(lkb, ms);
4245
4246        error = receive_unlock_args(ls, lkb, ms);
4247        if (error) {
4248                send_unlock_reply(r, lkb, error);
4249                goto out;
4250        }
4251
4252        error = do_unlock(r, lkb);
4253        send_unlock_reply(r, lkb, error);
4254        do_unlock_effects(r, lkb, error);
4255 out:
4256        unlock_rsb(r);
4257        put_rsb(r);
4258        dlm_put_lkb(lkb);
4259        return 0;
4260
4261 fail:
4262        setup_stub_lkb(ls, ms);
4263        send_unlock_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
4264        return error;
4265}
4266
4267static int receive_cancel(struct dlm_ls *ls, struct dlm_message *ms)
4268{
4269        struct dlm_lkb *lkb;
4270        struct dlm_rsb *r;
4271        int error;
4272
4273        error = find_lkb(ls, ms->m_remid, &lkb);
4274        if (error)
4275                goto fail;
4276
4277        receive_flags(lkb, ms);
4278
4279        r = lkb->lkb_resource;
4280
4281        hold_rsb(r);
4282        lock_rsb(r);
4283
4284        error = validate_message(lkb, ms);
4285        if (error)
4286                goto out;
4287
4288        error = do_cancel(r, lkb);
4289        send_cancel_reply(r, lkb, error);
4290        do_cancel_effects(r, lkb, error);
4291 out:
4292        unlock_rsb(r);
4293        put_rsb(r);
4294        dlm_put_lkb(lkb);
4295        return 0;
4296
4297 fail:
4298        setup_stub_lkb(ls, ms);
4299        send_cancel_reply(&ls->ls_stub_rsb, &ls->ls_stub_lkb, error);
4300        return error;
4301}
4302
4303static int receive_grant(struct dlm_ls *ls, struct dlm_message *ms)
4304{
4305        struct dlm_lkb *lkb;
4306        struct dlm_rsb *r;
4307        int error;
4308
4309        error = find_lkb(ls, ms->m_remid, &lkb);
4310        if (error)
4311                return error;
4312
4313        r = lkb->lkb_resource;
4314
4315        hold_rsb(r);
4316        lock_rsb(r);
4317
4318        error = validate_message(lkb, ms);
4319        if (error)
4320                goto out;
4321
4322        receive_flags_reply(lkb, ms);
4323        if (is_altmode(lkb))
4324                munge_altmode(lkb, ms);
4325        grant_lock_pc(r, lkb, ms);
4326        queue_cast(r, lkb, 0);
4327 out:
4328        unlock_rsb(r);
4329        put_rsb(r);
4330        dlm_put_lkb(lkb);
4331        return 0;
4332}
4333
4334static int receive_bast(struct dlm_ls *ls, struct dlm_message *ms)
4335{
4336        struct dlm_lkb *lkb;
4337        struct dlm_rsb *r;
4338        int error;
4339
4340        error = find_lkb(ls, ms->m_remid, &lkb);
4341        if (error)
4342                return error;
4343
4344        r = lkb->lkb_resource;
4345
4346        hold_rsb(r);
4347        lock_rsb(r);
4348
4349        error = validate_message(lkb, ms);
4350        if (error)
4351                goto out;
4352
4353        queue_bast(r, lkb, ms->m_bastmode);
4354        lkb->lkb_highbast = ms->m_bastmode;
4355 out:
4356        unlock_rsb(r);
4357        put_rsb(r);
4358        dlm_put_lkb(lkb);
4359        return 0;
4360}
4361
4362static void receive_lookup(struct dlm_ls *ls, struct dlm_message *ms)
4363{
4364        int len, error, ret_nodeid, from_nodeid, our_nodeid;
4365
4366        from_nodeid = ms->m_header.h_nodeid;
4367        our_nodeid = dlm_our_nodeid();
4368
4369        len = receive_extralen(ms);
4370
4371        error = dlm_master_lookup(ls, from_nodeid, ms->m_extra, len, 0,
4372                                  &ret_nodeid, NULL);
4373
4374        /* Optimization: we're master so treat lookup as a request */
4375        if (!error && ret_nodeid == our_nodeid) {
4376                receive_request(ls, ms);
4377                return;
4378        }
4379        send_lookup_reply(ls, ms, ret_nodeid, error);
4380}
4381
4382static void receive_remove(struct dlm_ls *ls, struct dlm_message *ms)
4383{
4384        char name[DLM_RESNAME_MAXLEN+1];
4385        struct dlm_rsb *r;
4386        uint32_t hash, b;
4387        int rv, len, dir_nodeid, from_nodeid;
4388
4389        from_nodeid = ms->m_header.h_nodeid;
4390
4391        len = receive_extralen(ms);
4392
4393        if (len > DLM_RESNAME_MAXLEN) {
4394                log_error(ls, "receive_remove from %d bad len %d",
4395                          from_nodeid, len);
4396                return;
4397        }
4398
4399        dir_nodeid = dlm_hash2nodeid(ls, ms->m_hash);
4400        if (dir_nodeid != dlm_our_nodeid()) {
4401                log_error(ls, "receive_remove from %d bad nodeid %d",
4402                          from_nodeid, dir_nodeid);
4403                return;
4404        }
4405
4406        /* Look for name on rsbtbl.toss, if it's there, kill it.
4407           If it's on rsbtbl.keep, it's being used, and we should ignore this
4408           message.  This is an expected race between the dir node sending a
4409           request to the master node at the same time as the master node sends
4410           a remove to the dir node.  The resolution to that race is for the
4411           dir node to ignore the remove message, and the master node to
4412           recreate the master rsb when it gets a request from the dir node for
4413           an rsb it doesn't have. */
4414
4415        memset(name, 0, sizeof(name));
4416        memcpy(name, ms->m_extra, len);
4417
4418        hash = jhash(name, len, 0);
4419        b = hash & (ls->ls_rsbtbl_size - 1);
4420
4421        spin_lock(&ls->ls_rsbtbl[b].lock);
4422
4423        rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].toss, name, len, &r);
4424        if (rv) {
4425                /* verify the rsb is on keep list per comment above */
4426                rv = dlm_search_rsb_tree(&ls->ls_rsbtbl[b].keep, name, len, &r);
4427                if (rv) {
4428                        /* should not happen */
4429                        log_error(ls, "receive_remove from %d not found %s",
4430                                  from_nodeid, name);
4431                        spin_unlock(&ls->ls_rsbtbl[b].lock);
4432                        return;
4433                }
4434                if (r->res_master_nodeid != from_nodeid) {
4435                        /* should not happen */
4436                        log_error(ls, "receive_remove keep from %d master %d",
4437                                  from_nodeid, r->res_master_nodeid);
4438                        dlm_print_rsb(r);
4439                        spin_unlock(&ls->ls_rsbtbl[b].lock);
4440                        return;
4441                }
4442
4443                log_debug(ls, "receive_remove from %d master %d first %x %s",
4444                          from_nodeid, r->res_master_nodeid, r->res_first_lkid,
4445                          name);
4446                spin_unlock(&ls->ls_rsbtbl[b].lock);
4447                return;
4448        }
4449
4450        if (r->res_master_nodeid != from_nodeid) {
4451                log_error(ls, "receive_remove toss from %d master %d",
4452                          from_nodeid, r->res_master_nodeid);
4453                dlm_print_rsb(r);
4454                spin_unlock(&ls->ls_rsbtbl[b].lock);
4455                return;
4456        }
4457
4458        if (kref_put(&r->res_ref, kill_rsb)) {
4459                rb_erase(&r->res_hashnode, &ls->ls_rsbtbl[b].toss);
4460                spin_unlock(&ls->ls_rsbtbl[b].lock);
4461                dlm_free_rsb(r);
4462        } else {
4463                log_error(ls, "receive_remove from %d rsb ref error",
4464                          from_nodeid);
4465                dlm_print_rsb(r);
4466                spin_unlock(&ls->ls_rsbtbl[b].lock);
4467        }
4468}
4469
4470static void receive_purge(struct dlm_ls *ls, struct dlm_message *ms)
4471{
4472        do_purge(ls, ms->m_nodeid, ms->m_pid);
4473}
4474
4475static int receive_request_reply(struct dlm_ls *ls, struct dlm_message *ms)
4476{
4477        struct dlm_lkb *lkb;
4478        struct dlm_rsb *r;
4479        int error, mstype, result;
4480        int from_nodeid = ms->m_header.h_nodeid;
4481
4482        error = find_lkb(ls, ms->m_remid, &lkb);
4483        if (error)
4484                return error;
4485
4486        r = lkb->lkb_resource;
4487        hold_rsb(r);
4488        lock_rsb(r);
4489
4490        error = validate_message(lkb, ms);
4491        if (error)
4492                goto out;
4493
4494        mstype = lkb->lkb_wait_type;
4495        error = remove_from_waiters(lkb, DLM_MSG_REQUEST_REPLY);
4496        if (error) {
4497                log_error(ls, "receive_request_reply %x remote %d %x result %d",
4498                          lkb->lkb_id, from_nodeid, ms->m_lkid, ms->m_result);
4499                dlm_dump_rsb(r);
4500                goto out;
4501        }
4502
4503        /* Optimization: the dir node was also the master, so it took our
4504           lookup as a request and sent request reply instead of lookup reply */
4505        if (mstype == DLM_MSG_LOOKUP) {
4506                r->res_master_nodeid = from_nodeid;
4507                r->res_nodeid = from_nodeid;
4508                lkb->lkb_nodeid = from_nodeid;
4509        }
4510
4511        /* this is the value returned from do_request() on the master */
4512        result = ms->m_result;
4513
4514        switch (result) {
4515        case -EAGAIN:
4516                /* request would block (be queued) on remote master */
4517                queue_cast(r, lkb, -EAGAIN);
4518                confirm_master(r, -EAGAIN);
4519                unhold_lkb(lkb); /* undoes create_lkb() */
4520                break;
4521
4522        case -EINPROGRESS:
4523        case 0:
4524                /* request was queued or granted on remote master */
4525                receive_flags_reply(lkb, ms);
4526                lkb->lkb_remid = ms->m_lkid;
4527                if (is_altmode(lkb))
4528                        munge_altmode(lkb, ms);
4529                if (result) {
4530                        add_lkb(r, lkb, DLM_LKSTS_WAITING);
4531                        add_timeout(lkb);
4532                } else {
4533                        grant_lock_pc(r, lkb, ms);
4534                        queue_cast(r, lkb, 0);
4535                }
4536                confirm_master(r, result);
4537                break;
4538
4539        case -EBADR:
4540        case -ENOTBLK:
4541                /* find_rsb failed to find rsb or rsb wasn't master */
4542                log_limit(ls, "receive_request_reply %x from %d %d "
4543                          "master %d dir %d first %x %s", lkb->lkb_id,
4544                          from_nodeid, result, r->res_master_nodeid,
4545                          r->res_dir_nodeid, r->res_first_lkid, r->res_name);
4546
4547                if (r->res_dir_nodeid != dlm_our_nodeid() &&
4548                    r->res_master_nodeid != dlm_our_nodeid()) {
4549                        /* cause _request_lock->set_master->send_lookup */
4550                        r->res_master_nodeid = 0;
4551                        r->res_nodeid = -1;
4552                        lkb->lkb_nodeid = -1;
4553                }
4554
4555                if (is_overlap(lkb)) {
4556                        /* we'll ignore error in cancel/unlock reply */
4557                        queue_cast_overlap(r, lkb);
4558                        confirm_master(r, result);
4559                        unhold_lkb(lkb); /* undoes create_lkb() */
4560                } else {
4561                        _request_lock(r, lkb);
4562
4563                        if (r->res_master_nodeid == dlm_our_nodeid())
4564                                confirm_master(r, 0);
4565                }
4566                break;
4567
4568        default:
4569                log_error(ls, "receive_request_reply %x error %d",
4570                          lkb->lkb_id, result);
4571        }
4572
4573        if (is_overlap_unlock(lkb) && (result == 0 || result == -EINPROGRESS)) {
4574                log_debug(ls, "receive_request_reply %x result %d unlock",
4575                          lkb->lkb_id, result);
4576                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
4577                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
4578                send_unlock(r, lkb);
4579        } else if (is_overlap_cancel(lkb) && (result == -EINPROGRESS)) {
4580                log_debug(ls, "receive_request_reply %x cancel", lkb->lkb_id);
4581                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
4582                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
4583                send_cancel(r, lkb);
4584        } else {
4585                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
4586                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
4587        }
4588 out:
4589        unlock_rsb(r);
4590        put_rsb(r);
4591        dlm_put_lkb(lkb);
4592        return 0;
4593}
4594
4595static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
4596                                    struct dlm_message *ms)
4597{
4598        /* this is the value returned from do_convert() on the master */
4599        switch (ms->m_result) {
4600        case -EAGAIN:
4601                /* convert would block (be queued) on remote master */
4602                queue_cast(r, lkb, -EAGAIN);
4603                break;
4604
4605        case -EDEADLK:
4606                receive_flags_reply(lkb, ms);
4607                revert_lock_pc(r, lkb);
4608                queue_cast(r, lkb, -EDEADLK);
4609                break;
4610
4611        case -EINPROGRESS:
4612                /* convert was queued on remote master */
4613                receive_flags_reply(lkb, ms);
4614                if (is_demoted(lkb))
4615                        munge_demoted(lkb);
4616                del_lkb(r, lkb);
4617                add_lkb(r, lkb, DLM_LKSTS_CONVERT);
4618                add_timeout(lkb);
4619                break;
4620
4621        case 0:
4622                /* convert was granted on remote master */
4623                receive_flags_reply(lkb, ms);
4624                if (is_demoted(lkb))
4625                        munge_demoted(lkb);
4626                grant_lock_pc(r, lkb, ms);
4627                queue_cast(r, lkb, 0);
4628                break;
4629
4630        default:
4631                log_error(r->res_ls, "receive_convert_reply %x remote %d %x %d",
4632                          lkb->lkb_id, ms->m_header.h_nodeid, ms->m_lkid,
4633                          ms->m_result);
4634                dlm_print_rsb(r);
4635                dlm_print_lkb(lkb);
4636        }
4637}
4638
4639static void _receive_convert_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
4640{
4641        struct dlm_rsb *r = lkb->lkb_resource;
4642        int error;
4643
4644        hold_rsb(r);
4645        lock_rsb(r);
4646
4647        error = validate_message(lkb, ms);
4648        if (error)
4649                goto out;
4650
4651        /* stub reply can happen with waiters_mutex held */
4652        error = remove_from_waiters_ms(lkb, ms);
4653        if (error)
4654                goto out;
4655
4656        __receive_convert_reply(r, lkb, ms);
4657 out:
4658        unlock_rsb(r);
4659        put_rsb(r);
4660}
4661
4662static int receive_convert_reply(struct dlm_ls *ls, struct dlm_message *ms)
4663{
4664        struct dlm_lkb *lkb;
4665        int error;
4666
4667        error = find_lkb(ls, ms->m_remid, &lkb);
4668        if (error)
4669                return error;
4670
4671        _receive_convert_reply(lkb, ms);
4672        dlm_put_lkb(lkb);
4673        return 0;
4674}
4675
4676static void _receive_unlock_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
4677{
4678        struct dlm_rsb *r = lkb->lkb_resource;
4679        int error;
4680
4681        hold_rsb(r);
4682        lock_rsb(r);
4683
4684        error = validate_message(lkb, ms);
4685        if (error)
4686                goto out;
4687
4688        /* stub reply can happen with waiters_mutex held */
4689        error = remove_from_waiters_ms(lkb, ms);
4690        if (error)
4691                goto out;
4692
4693        /* this is the value returned from do_unlock() on the master */
4694
4695        switch (ms->m_result) {
4696        case -DLM_EUNLOCK:
4697                receive_flags_reply(lkb, ms);
4698                remove_lock_pc(r, lkb);
4699                queue_cast(r, lkb, -DLM_EUNLOCK);
4700                break;
4701        case -ENOENT:
4702                break;
4703        default:
4704                log_error(r->res_ls, "receive_unlock_reply %x error %d",
4705                          lkb->lkb_id, ms->m_result);
4706        }
4707 out:
4708        unlock_rsb(r);
4709        put_rsb(r);
4710}
4711
4712static int receive_unlock_reply(struct dlm_ls *ls, struct dlm_message *ms)
4713{
4714        struct dlm_lkb *lkb;
4715        int error;
4716
4717        error = find_lkb(ls, ms->m_remid, &lkb);
4718        if (error)
4719                return error;
4720
4721        _receive_unlock_reply(lkb, ms);
4722        dlm_put_lkb(lkb);
4723        return 0;
4724}
4725
4726static void _receive_cancel_reply(struct dlm_lkb *lkb, struct dlm_message *ms)
4727{
4728        struct dlm_rsb *r = lkb->lkb_resource;
4729        int error;
4730
4731        hold_rsb(r);
4732        lock_rsb(r);
4733
4734        error = validate_message(lkb, ms);
4735        if (error)
4736                goto out;
4737
4738        /* stub reply can happen with waiters_mutex held */
4739        error = remove_from_waiters_ms(lkb, ms);
4740        if (error)
4741                goto out;
4742
4743        /* this is the value returned from do_cancel() on the master */
4744
4745        switch (ms->m_result) {
4746        case -DLM_ECANCEL:
4747                receive_flags_reply(lkb, ms);
4748                revert_lock_pc(r, lkb);
4749                queue_cast(r, lkb, -DLM_ECANCEL);
4750                break;
4751        case 0:
4752                break;
4753        default:
4754                log_error(r->res_ls, "receive_cancel_reply %x error %d",
4755                          lkb->lkb_id, ms->m_result);
4756        }
4757 out:
4758        unlock_rsb(r);
4759        put_rsb(r);
4760}
4761
4762static int receive_cancel_reply(struct dlm_ls *ls, struct dlm_message *ms)
4763{
4764        struct dlm_lkb *lkb;
4765        int error;
4766
4767        error = find_lkb(ls, ms->m_remid, &lkb);
4768        if (error)
4769                return error;
4770
4771        _receive_cancel_reply(lkb, ms);
4772        dlm_put_lkb(lkb);
4773        return 0;
4774}
4775
4776static void receive_lookup_reply(struct dlm_ls *ls, struct dlm_message *ms)
4777{
4778        struct dlm_lkb *lkb;
4779        struct dlm_rsb *r;
4780        int error, ret_nodeid;
4781        int do_lookup_list = 0;
4782
4783        error = find_lkb(ls, ms->m_lkid, &lkb);
4784        if (error) {
4785                log_error(ls, "receive_lookup_reply no lkid %x", ms->m_lkid);
4786                return;
4787        }
4788
4789        /* ms->m_result is the value returned by dlm_master_lookup on dir node
4790           FIXME: will a non-zero error ever be returned? */
4791
4792        r = lkb->lkb_resource;
4793        hold_rsb(r);
4794        lock_rsb(r);
4795
4796        error = remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY);
4797        if (error)
4798                goto out;
4799
4800        ret_nodeid = ms->m_nodeid;
4801
4802        /* We sometimes receive a request from the dir node for this
4803           rsb before we've received the dir node's loookup_reply for it.
4804           The request from the dir node implies we're the master, so we set
4805           ourself as master in receive_request_reply, and verify here that
4806           we are indeed the master. */
4807
4808        if (r->res_master_nodeid && (r->res_master_nodeid != ret_nodeid)) {
4809                /* This should never happen */
4810                log_error(ls, "receive_lookup_reply %x from %d ret %d "
4811                          "master %d dir %d our %d first %x %s",
4812                          lkb->lkb_id, ms->m_header.h_nodeid, ret_nodeid,
4813                          r->res_master_nodeid, r->res_dir_nodeid,
4814                          dlm_our_nodeid(), r->res_first_lkid, r->res_name);
4815        }
4816
4817        if (ret_nodeid == dlm_our_nodeid()) {
4818                r->res_master_nodeid = ret_nodeid;
4819                r->res_nodeid = 0;
4820                do_lookup_list = 1;
4821                r->res_first_lkid = 0;
4822        } else if (ret_nodeid == -1) {
4823                /* the remote node doesn't believe it's the dir node */
4824                log_error(ls, "receive_lookup_reply %x from %d bad ret_nodeid",
4825                          lkb->lkb_id, ms->m_header.h_nodeid);
4826                r->res_master_nodeid = 0;
4827                r->res_nodeid = -1;
4828                lkb->lkb_nodeid = -1;
4829        } else {
4830                /* set_master() will set lkb_nodeid from r */
4831                r->res_master_nodeid = ret_nodeid;
4832                r->res_nodeid = ret_nodeid;
4833        }
4834
4835        if (is_overlap(lkb)) {
4836                log_debug(ls, "receive_lookup_reply %x unlock %x",
4837                          lkb->lkb_id, lkb->lkb_flags);
4838                queue_cast_overlap(r, lkb);
4839                unhold_lkb(lkb); /* undoes create_lkb() */
4840                goto out_list;
4841        }
4842
4843        _request_lock(r, lkb);
4844
4845 out_list:
4846        if (do_lookup_list)
4847                process_lookup_list(r);
4848 out:
4849        unlock_rsb(r);
4850        put_rsb(r);
4851        dlm_put_lkb(lkb);
4852}
4853
4854static void _receive_message(struct dlm_ls *ls, struct dlm_message *ms,
4855                             uint32_t saved_seq)
4856{
4857        int error = 0, noent = 0;
4858
4859        if (!dlm_is_member(ls, ms->m_header.h_nodeid)) {
4860                log_limit(ls, "receive %d from non-member %d %x %x %d",
4861                          ms->m_type, ms->m_header.h_nodeid, ms->m_lkid,
4862                          ms->m_remid, ms->m_result);
4863                return;
4864        }
4865
4866        switch (ms->m_type) {
4867
4868        /* messages sent to a master node */
4869
4870        case DLM_MSG_REQUEST:
4871                error = receive_request(ls, ms);
4872                break;
4873
4874        case DLM_MSG_CONVERT:
4875                error = receive_convert(ls, ms);
4876                break;
4877
4878        case DLM_MSG_UNLOCK:
4879                error = receive_unlock(ls, ms);
4880                break;
4881
4882        case DLM_MSG_CANCEL:
4883                noent = 1;
4884                error = receive_cancel(ls, ms);
4885                break;
4886
4887        /* messages sent from a master node (replies to above) */
4888
4889        case DLM_MSG_REQUEST_REPLY:
4890                error = receive_request_reply(ls, ms);
4891                break;
4892
4893        case DLM_MSG_CONVERT_REPLY:
4894                error = receive_convert_reply(ls, ms);
4895                break;
4896
4897        case DLM_MSG_UNLOCK_REPLY:
4898                error = receive_unlock_reply(ls, ms);
4899                break;
4900
4901        case DLM_MSG_CANCEL_REPLY:
4902                error = receive_cancel_reply(ls, ms);
4903                break;
4904
4905        /* messages sent from a master node (only two types of async msg) */
4906
4907        case DLM_MSG_GRANT:
4908                noent = 1;
4909                error = receive_grant(ls, ms);
4910                break;
4911
4912        case DLM_MSG_BAST:
4913                noent = 1;
4914                error = receive_bast(ls, ms);
4915                break;
4916
4917        /* messages sent to a dir node */
4918
4919        case DLM_MSG_LOOKUP:
4920                receive_lookup(ls, ms);
4921                break;
4922
4923        case DLM_MSG_REMOVE:
4924                receive_remove(ls, ms);
4925                break;
4926
4927        /* messages sent from a dir node (remove has no reply) */
4928
4929        case DLM_MSG_LOOKUP_REPLY:
4930                receive_lookup_reply(ls, ms);
4931                break;
4932
4933        /* other messages */
4934
4935        case DLM_MSG_PURGE:
4936                receive_purge(ls, ms);
4937                break;
4938
4939        default:
4940                log_error(ls, "unknown message type %d", ms->m_type);
4941        }
4942
4943        /*
4944         * When checking for ENOENT, we're checking the result of
4945         * find_lkb(m_remid):
4946         *
4947         * The lock id referenced in the message wasn't found.  This may
4948         * happen in normal usage for the async messages and cancel, so
4949         * only use log_debug for them.
4950         *
4951         * Some errors are expected and normal.
4952         */
4953
4954        if (error == -ENOENT && noent) {
4955                log_debug(ls, "receive %d no %x remote %d %x saved_seq %u",
4956                          ms->m_type, ms->m_remid, ms->m_header.h_nodeid,
4957                          ms->m_lkid, saved_seq);
4958        } else if (error == -ENOENT) {
4959                log_error(ls, "receive %d no %x remote %d %x saved_seq %u",
4960                          ms->m_type, ms->m_remid, ms->m_header.h_nodeid,
4961                          ms->m_lkid, saved_seq);
4962
4963                if (ms->m_type == DLM_MSG_CONVERT)
4964                        dlm_dump_rsb_hash(ls, ms->m_hash);
4965        }
4966
4967        if (error == -EINVAL) {
4968                log_error(ls, "receive %d inval from %d lkid %x remid %x "
4969                          "saved_seq %u",
4970                          ms->m_type, ms->m_header.h_nodeid,
4971                          ms->m_lkid, ms->m_remid, saved_seq);
4972        }
4973}
4974
4975/* If the lockspace is in recovery mode (locking stopped), then normal
4976   messages are saved on the requestqueue for processing after recovery is
4977   done.  When not in recovery mode, we wait for dlm_recoverd to drain saved
4978   messages off the requestqueue before we process new ones. This occurs right
4979   after recovery completes when we transition from saving all messages on
4980   requestqueue, to processing all the saved messages, to processing new
4981   messages as they arrive. */
4982
4983static void dlm_receive_message(struct dlm_ls *ls, struct dlm_message *ms,
4984                                int nodeid)
4985{
4986        if (dlm_locking_stopped(ls)) {
4987                /* If we were a member of this lockspace, left, and rejoined,
4988                   other nodes may still be sending us messages from the
4989                   lockspace generation before we left. */
4990                if (!ls->ls_generation) {
4991                        log_limit(ls, "receive %d from %d ignore old gen",
4992                                  ms->m_type, nodeid);
4993                        return;
4994                }
4995
4996                dlm_add_requestqueue(ls, nodeid, ms);
4997        } else {
4998                dlm_wait_requestqueue(ls);
4999                _receive_message(ls, ms, 0);
5000        }
5001}
5002
5003/* This is called by dlm_recoverd to process messages that were saved on
5004   the requestqueue. */
5005
5006void dlm_receive_message_saved(struct dlm_ls *ls, struct dlm_message *ms,
5007                               uint32_t saved_seq)
5008{
5009        _receive_message(ls, ms, saved_seq);
5010}
5011
5012/* This is called by the midcomms layer when something is received for
5013   the lockspace.  It could be either a MSG (normal message sent as part of
5014   standard locking activity) or an RCOM (recovery message sent as part of
5015   lockspace recovery). */
5016
5017void dlm_receive_buffer(union dlm_packet *p, int nodeid)
5018{
5019        struct dlm_header *hd = &p->header;
5020        struct dlm_ls *ls;
5021        int type = 0;
5022
5023        switch (hd->h_cmd) {
5024        case DLM_MSG:
5025                dlm_message_in(&p->message);
5026                type = p->message.m_type;
5027                break;
5028        case DLM_RCOM:
5029                dlm_rcom_in(&p->rcom);
5030                type = p->rcom.rc_type;
5031                break;
5032        default:
5033                log_print("invalid h_cmd %d from %u", hd->h_cmd, nodeid);
5034                return;
5035        }
5036
5037        if (hd->h_nodeid != nodeid) {
5038                log_print("invalid h_nodeid %d from %d lockspace %x",
5039                          hd->h_nodeid, nodeid, hd->h_lockspace);
5040                return;
5041        }
5042
5043        ls = dlm_find_lockspace_global(hd->h_lockspace);
5044        if (!ls) {
5045                if (dlm_config.ci_log_debug) {
5046                        printk_ratelimited(KERN_DEBUG "dlm: invalid lockspace "
5047                                "%u from %d cmd %d type %d\n",
5048                                hd->h_lockspace, nodeid, hd->h_cmd, type);
5049                }
5050
5051                if (hd->h_cmd == DLM_RCOM && type == DLM_RCOM_STATUS)
5052                        dlm_send_ls_not_ready(nodeid, &p->rcom);
5053                return;
5054        }
5055
5056        /* this rwsem allows dlm_ls_stop() to wait for all dlm_recv threads to
5057           be inactive (in this ls) before transitioning to recovery mode */
5058
5059        down_read(&ls->ls_recv_active);
5060        if (hd->h_cmd == DLM_MSG)
5061                dlm_receive_message(ls, &p->message, nodeid);
5062        else
5063                dlm_receive_rcom(ls, &p->rcom, nodeid);
5064        up_read(&ls->ls_recv_active);
5065
5066        dlm_put_lockspace(ls);
5067}
5068
5069static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb,
5070                                   struct dlm_message *ms_stub)
5071{
5072        if (middle_conversion(lkb)) {
5073                hold_lkb(lkb);
5074                memset(ms_stub, 0, sizeof(struct dlm_message));
5075                ms_stub->m_flags = DLM_IFL_STUB_MS;
5076                ms_stub->m_type = DLM_MSG_CONVERT_REPLY;
5077                ms_stub->m_result = -EINPROGRESS;
5078                ms_stub->m_header.h_nodeid = lkb->lkb_nodeid;
5079                _receive_convert_reply(lkb, ms_stub);
5080
5081                /* Same special case as in receive_rcom_lock_args() */
5082                lkb->lkb_grmode = DLM_LOCK_IV;
5083                rsb_set_flag(lkb->lkb_resource, RSB_RECOVER_CONVERT);
5084                unhold_lkb(lkb);
5085
5086        } else if (lkb->lkb_rqmode >= lkb->lkb_grmode) {
5087                lkb->lkb_flags |= DLM_IFL_RESEND;
5088        }
5089
5090        /* lkb->lkb_rqmode < lkb->lkb_grmode shouldn't happen since down
5091           conversions are async; there's no reply from the remote master */
5092}
5093
5094/* A waiting lkb needs recovery if the master node has failed, or
5095   the master node is changing (only when no directory is used) */
5096
5097static int waiter_needs_recovery(struct dlm_ls *ls, struct dlm_lkb *lkb,
5098                                 int dir_nodeid)
5099{
5100        if (dlm_no_directory(ls))
5101                return 1;
5102
5103        if (dlm_is_removed(ls, lkb->lkb_wait_nodeid))
5104                return 1;
5105
5106        return 0;
5107}
5108
5109/* Recovery for locks that are waiting for replies from nodes that are now
5110   gone.  We can just complete unlocks and cancels by faking a reply from the
5111   dead node.  Requests and up-conversions we flag to be resent after
5112   recovery.  Down-conversions can just be completed with a fake reply like
5113   unlocks.  Conversions between PR and CW need special attention. */
5114
5115void dlm_recover_waiters_pre(struct dlm_ls *ls)
5116{
5117        struct dlm_lkb *lkb, *safe;
5118        struct dlm_message *ms_stub;
5119        int wait_type, stub_unlock_result, stub_cancel_result;
5120        int dir_nodeid;
5121
5122        ms_stub = kmalloc(sizeof(struct dlm_message), GFP_KERNEL);
5123        if (!ms_stub) {
5124                log_error(ls, "dlm_recover_waiters_pre no mem");
5125                return;
5126        }
5127
5128        mutex_lock(&ls->ls_waiters_mutex);
5129
5130        list_for_each_entry_safe(lkb, safe, &ls->ls_waiters, lkb_wait_reply) {
5131
5132                dir_nodeid = dlm_dir_nodeid(lkb->lkb_resource);
5133
5134                /* exclude debug messages about unlocks because there can be so
5135                   many and they aren't very interesting */
5136
5137                if (lkb->lkb_wait_type != DLM_MSG_UNLOCK) {
5138                        log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d "
5139                                  "lkb_nodeid %d wait_nodeid %d dir_nodeid %d",
5140                                  lkb->lkb_id,
5141                                  lkb->lkb_remid,
5142                                  lkb->lkb_wait_type,
5143                                  lkb->lkb_resource->res_nodeid,
5144                                  lkb->lkb_nodeid,
5145                                  lkb->lkb_wait_nodeid,
5146                                  dir_nodeid);
5147                }
5148
5149                /* all outstanding lookups, regardless of destination  will be
5150                   resent after recovery is done */
5151
5152                if (lkb->lkb_wait_type == DLM_MSG_LOOKUP) {
5153                        lkb->lkb_flags |= DLM_IFL_RESEND;
5154                        continue;
5155                }
5156
5157                if (!waiter_needs_recovery(ls, lkb, dir_nodeid))
5158                        continue;
5159
5160                wait_type = lkb->lkb_wait_type;
5161                stub_unlock_result = -DLM_EUNLOCK;
5162                stub_cancel_result = -DLM_ECANCEL;
5163
5164                /* Main reply may have been received leaving a zero wait_type,
5165                   but a reply for the overlapping op may not have been
5166                   received.  In that case we need to fake the appropriate
5167                   reply for the overlap op. */
5168
5169                if (!wait_type) {
5170                        if (is_overlap_cancel(lkb)) {
5171                                wait_type = DLM_MSG_CANCEL;
5172                                if (lkb->lkb_grmode == DLM_LOCK_IV)
5173                                        stub_cancel_result = 0;
5174                        }
5175                        if (is_overlap_unlock(lkb)) {
5176                                wait_type = DLM_MSG_UNLOCK;
5177                                if (lkb->lkb_grmode == DLM_LOCK_IV)
5178                                        stub_unlock_result = -ENOENT;
5179                        }
5180
5181                        log_debug(ls, "rwpre overlap %x %x %d %d %d",
5182                                  lkb->lkb_id, lkb->lkb_flags, wait_type,
5183                                  stub_cancel_result, stub_unlock_result);
5184                }
5185
5186                switch (wait_type) {
5187
5188                case DLM_MSG_REQUEST:
5189                        lkb->lkb_flags |= DLM_IFL_RESEND;
5190                        break;
5191
5192                case DLM_MSG_CONVERT:
5193                        recover_convert_waiter(ls, lkb, ms_stub);
5194                        break;
5195
5196                case DLM_MSG_UNLOCK:
5197                        hold_lkb(lkb);
5198                        memset(ms_stub, 0, sizeof(struct dlm_message));
5199                        ms_stub->m_flags = DLM_IFL_STUB_MS;
5200                        ms_stub->m_type = DLM_MSG_UNLOCK_REPLY;
5201                        ms_stub->m_result = stub_unlock_result;
5202                        ms_stub->m_header.h_nodeid = lkb->lkb_nodeid;
5203                        _receive_unlock_reply(lkb, ms_stub);
5204                        dlm_put_lkb(lkb);
5205                        break;
5206
5207                case DLM_MSG_CANCEL:
5208                        hold_lkb(lkb);
5209                        memset(ms_stub, 0, sizeof(struct dlm_message));
5210                        ms_stub->m_flags = DLM_IFL_STUB_MS;
5211                        ms_stub->m_type = DLM_MSG_CANCEL_REPLY;
5212                        ms_stub->m_result = stub_cancel_result;
5213                        ms_stub->m_header.h_nodeid = lkb->lkb_nodeid;
5214                        _receive_cancel_reply(lkb, ms_stub);
5215                        dlm_put_lkb(lkb);
5216                        break;
5217
5218                default:
5219                        log_error(ls, "invalid lkb wait_type %d %d",
5220                                  lkb->lkb_wait_type, wait_type);
5221                }
5222                schedule();
5223        }
5224        mutex_unlock(&ls->ls_waiters_mutex);
5225        kfree(ms_stub);
5226}
5227
5228static struct dlm_lkb *find_resend_waiter(struct dlm_ls *ls)
5229{
5230        struct dlm_lkb *lkb;
5231        int found = 0;
5232
5233        mutex_lock(&ls->ls_waiters_mutex);
5234        list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) {
5235                if (lkb->lkb_flags & DLM_IFL_RESEND) {
5236                        hold_lkb(lkb);
5237                        found = 1;
5238                        break;
5239                }
5240        }
5241        mutex_unlock(&ls->ls_waiters_mutex);
5242
5243        if (!found)
5244                lkb = NULL;
5245        return lkb;
5246}
5247
5248/* Deal with lookups and lkb's marked RESEND from _pre.  We may now be the
5249   master or dir-node for r.  Processing the lkb may result in it being placed
5250   back on waiters. */
5251
5252/* We do this after normal locking has been enabled and any saved messages
5253   (in requestqueue) have been processed.  We should be confident that at
5254   this point we won't get or process a reply to any of these waiting
5255   operations.  But, new ops may be coming in on the rsbs/locks here from
5256   userspace or remotely. */
5257
5258/* there may have been an overlap unlock/cancel prior to recovery or after
5259   recovery.  if before, the lkb may still have a pos wait_count; if after, the
5260   overlap flag would just have been set and nothing new sent.  we can be
5261   confident here than any replies to either the initial op or overlap ops
5262   prior to recovery have been received. */
5263
5264int dlm_recover_waiters_post(struct dlm_ls *ls)
5265{
5266        struct dlm_lkb *lkb;
5267        struct dlm_rsb *r;
5268        int error = 0, mstype, err, oc, ou;
5269
5270        while (1) {
5271                if (dlm_locking_stopped(ls)) {
5272                        log_debug(ls, "recover_waiters_post aborted");
5273                        error = -EINTR;
5274                        break;
5275                }
5276
5277                lkb = find_resend_waiter(ls);
5278                if (!lkb)
5279                        break;
5280
5281                r = lkb->lkb_resource;
5282                hold_rsb(r);
5283                lock_rsb(r);
5284
5285                mstype = lkb->lkb_wait_type;
5286                oc = is_overlap_cancel(lkb);
5287                ou = is_overlap_unlock(lkb);
5288                err = 0;
5289
5290                log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d "
5291                          "lkb_nodeid %d wait_nodeid %d dir_nodeid %d "
5292                          "overlap %d %d", lkb->lkb_id, lkb->lkb_remid, mstype,
5293                          r->res_nodeid, lkb->lkb_nodeid, lkb->lkb_wait_nodeid,
5294                          dlm_dir_nodeid(r), oc, ou);
5295
5296                /* At this point we assume that we won't get a reply to any
5297                   previous op or overlap op on this lock.  First, do a big
5298                   remove_from_waiters() for all previous ops. */
5299
5300                lkb->lkb_flags &= ~DLM_IFL_RESEND;
5301                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_UNLOCK;
5302                lkb->lkb_flags &= ~DLM_IFL_OVERLAP_CANCEL;
5303                lkb->lkb_wait_type = 0;
5304                lkb->lkb_wait_count = 0;
5305                mutex_lock(&ls->ls_waiters_mutex);
5306                list_del_init(&lkb->lkb_wait_reply);
5307                mutex_unlock(&ls->ls_waiters_mutex);
5308                unhold_lkb(lkb); /* for waiters list */
5309
5310                if (oc || ou) {
5311                        /* do an unlock or cancel instead of resending */
5312                        switch (mstype) {
5313                        case DLM_MSG_LOOKUP:
5314                        case DLM_MSG_REQUEST:
5315                                queue_cast(r, lkb, ou ? -DLM_EUNLOCK :
5316                                                        -DLM_ECANCEL);
5317                                unhold_lkb(lkb); /* undoes create_lkb() */
5318                                break;
5319                        case DLM_MSG_CONVERT:
5320                                if (oc) {
5321                                        queue_cast(r, lkb, -DLM_ECANCEL);
5322                                } else {
5323                                        lkb->lkb_exflags |= DLM_LKF_FORCEUNLOCK;
5324                                        _unlock_lock(r, lkb);
5325                                }
5326                                break;
5327                        default:
5328                                err = 1;
5329                        }
5330                } else {
5331                        switch (mstype) {
5332                        case DLM_MSG_LOOKUP:
5333                        case DLM_MSG_REQUEST:
5334                                _request_lock(r, lkb);
5335                                if (is_master(r))
5336                                        confirm_master(r, 0);
5337                                break;
5338                        case DLM_MSG_CONVERT:
5339                                _convert_lock(r, lkb);
5340                                break;
5341                        default:
5342                                err = 1;
5343                        }
5344                }
5345
5346                if (err) {
5347                        log_error(ls, "waiter %x msg %d r_nodeid %d "
5348                                  "dir_nodeid %d overlap %d %d",
5349                                  lkb->lkb_id, mstype, r->res_nodeid,
5350                                  dlm_dir_nodeid(r), oc, ou);
5351                }
5352                unlock_rsb(r);
5353                put_rsb(r);
5354                dlm_put_lkb(lkb);
5355        }
5356
5357        return error;
5358}
5359
5360static void purge_mstcpy_list(struct dlm_ls *ls, struct dlm_rsb *r,
5361                              struct list_head *list)
5362{
5363        struct dlm_lkb *lkb, *safe;
5364
5365        list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) {
5366                if (!is_master_copy(lkb))
5367                        continue;
5368
5369                /* don't purge lkbs we've added in recover_master_copy for
5370                   the current recovery seq */
5371
5372                if (lkb->lkb_recover_seq == ls->ls_recover_seq)
5373                        continue;
5374
5375                del_lkb(r, lkb);
5376
5377                /* this put should free the lkb */
5378                if (!dlm_put_lkb(lkb))
5379                        log_error(ls, "purged mstcpy lkb not released");
5380        }
5381}
5382
5383void dlm_purge_mstcpy_locks(struct dlm_rsb *r)
5384{
5385        struct dlm_ls *ls = r->res_ls;
5386
5387        purge_mstcpy_list(ls, r, &r->res_grantqueue);
5388        purge_mstcpy_list(ls, r, &r->res_convertqueue);
5389        purge_mstcpy_list(ls, r, &r->res_waitqueue);
5390}
5391
5392static void purge_dead_list(struct dlm_ls *ls, struct dlm_rsb *r,
5393                            struct list_head *list,
5394                            int nodeid_gone, unsigned int *count)
5395{
5396        struct dlm_lkb *lkb, *safe;
5397
5398        list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) {
5399                if (!is_master_copy(lkb))
5400                        continue;
5401
5402                if ((lkb->lkb_nodeid == nodeid_gone) ||
5403                    dlm_is_removed(ls, lkb->lkb_nodeid)) {
5404
5405                        /* tell recover_lvb to invalidate the lvb
5406                           because a node holding EX/PW failed */
5407                        if ((lkb->lkb_exflags & DLM_LKF_VALBLK) &&
5408                            (lkb->lkb_grmode >= DLM_LOCK_PW)) {
5409                                rsb_set_flag(r, RSB_RECOVER_LVB_INVAL);
5410                        }
5411
5412                        del_lkb(r, lkb);
5413
5414                        /* this put should free the lkb */
5415                        if (!dlm_put_lkb(lkb))
5416                                log_error(ls, "purged dead lkb not released");
5417
5418                        rsb_set_flag(r, RSB_RECOVER_GRANT);
5419
5420                        (*count)++;
5421                }
5422        }
5423}
5424
5425/* Get rid of locks held by nodes that are gone. */
5426
5427void dlm_recover_purge(struct dlm_ls *ls)
5428{
5429        struct dlm_rsb *r;
5430        struct dlm_member *memb;
5431        int nodes_count = 0;
5432        int nodeid_gone = 0;
5433        unsigned int lkb_count = 0;
5434
5435        /* cache one removed nodeid to optimize the common
5436           case of a single node removed */
5437
5438        list_for_each_entry(memb, &ls->ls_nodes_gone, list) {
5439                nodes_count++;
5440                nodeid_gone = memb->nodeid;
5441        }
5442
5443        if (!nodes_count)
5444                return;
5445
5446        down_write(&ls->ls_root_sem);
5447        list_for_each_entry(r, &ls->ls_root_list, res_root_list) {
5448                hold_rsb(r);
5449                lock_rsb(r);
5450                if (is_master(r)) {
5451                        purge_dead_list(ls, r, &r->res_grantqueue,
5452                                        nodeid_gone, &lkb_count);
5453                        purge_dead_list(ls, r, &r->res_convertqueue,
5454                                        nodeid_gone, &lkb_count);
5455                        purge_dead_list(ls, r, &r->res_waitqueue,
5456                                        nodeid_gone, &lkb_count);
5457                }
5458                unlock_rsb(r);
5459                unhold_rsb(r);
5460                cond_resched();
5461        }
5462        up_write(&ls->ls_root_sem);
5463
5464        if (lkb_count)
5465                log_debug(ls, "dlm_recover_purge %u locks for %u nodes",
5466                          lkb_count, nodes_count);
5467}
5468
5469static struct dlm_rsb *find_grant_rsb(struct dlm_ls *ls, int bucket)
5470{
5471        struct rb_node *n;
5472        struct dlm_rsb *r;
5473
5474        spin_lock(&ls->ls_rsbtbl[bucket].lock);
5475        for (n = rb_first(&ls->ls_rsbtbl[bucket].keep); n; n = rb_next(n)) {
5476                r = rb_entry(n, struct dlm_rsb, res_hashnode);
5477
5478                if (!rsb_flag(r, RSB_RECOVER_GRANT))
5479                        continue;
5480                if (!is_master(r)) {
5481                        rsb_clear_flag(r, RSB_RECOVER_GRANT);
5482                        continue;
5483                }
5484                hold_rsb(r);
5485                spin_unlock(&ls->ls_rsbtbl[bucket].lock);
5486                return r;
5487        }
5488        spin_unlock(&ls->ls_rsbtbl[bucket].lock);
5489        return NULL;
5490}
5491
5492/*
5493 * Attempt to grant locks on resources that we are the master of.
5494 * Locks may have become grantable during recovery because locks
5495 * from departed nodes have been purged (or not rebuilt), allowing
5496 * previously blocked locks to now be granted.  The subset of rsb's
5497 * we are interested in are those with lkb's on either the convert or
5498 * waiting queues.
5499 *
5500 * Simplest would be to go through each master rsb and check for non-empty
5501 * convert or waiting queues, and attempt to grant on those rsbs.
5502 * Checking the queues requires lock_rsb, though, for which we'd need
5503 * to release the rsbtbl lock.  This would make iterating through all
5504 * rsb's very inefficient.  So, we rely on earlier recovery routines
5505 * to set RECOVER_GRANT on any rsb's that we should attempt to grant
5506 * locks for.
5507 */
5508
5509void dlm_recover_grant(struct dlm_ls *ls)
5510{
5511        struct dlm_rsb *r;
5512        int bucket = 0;
5513        unsigned int count = 0;
5514        unsigned int rsb_count = 0;
5515        unsigned int lkb_count = 0;
5516
5517        while (1) {
5518                r = find_grant_rsb(ls, bucket);
5519                if (!r) {
5520                        if (bucket == ls->ls_rsbtbl_size - 1)
5521                                break;
5522                        bucket++;
5523                        continue;
5524                }
5525                rsb_count++;
5526                count = 0;
5527                lock_rsb(r);
5528                /* the RECOVER_GRANT flag is checked in the grant path */
5529                grant_pending_locks(r, &count);
5530                rsb_clear_flag(r, RSB_RECOVER_GRANT);
5531                lkb_count += count;
5532                confirm_master(r, 0);
5533                unlock_rsb(r);
5534                put_rsb(r);
5535                cond_resched();
5536        }
5537
5538        if (lkb_count)
5539                log_debug(ls, "dlm_recover_grant %u locks on %u resources",
5540                          lkb_count, rsb_count);
5541}
5542
5543static struct dlm_lkb *search_remid_list(struct list_head *head, int nodeid,
5544                                         uint32_t remid)
5545{
5546        struct dlm_lkb *lkb;
5547
5548        list_for_each_entry(lkb, head, lkb_statequeue) {
5549                if (lkb->lkb_nodeid == nodeid && lkb->lkb_remid == remid)
5550                        return lkb;
5551        }
5552        return NULL;
5553}
5554
5555static struct dlm_lkb *search_remid(struct dlm_rsb *r, int nodeid,
5556                                    uint32_t remid)
5557{
5558        struct dlm_lkb *lkb;
5559
5560        lkb = search_remid_list(&r->res_grantqueue, nodeid, remid);
5561        if (lkb)
5562                return lkb;
5563        lkb = search_remid_list(&r->res_convertqueue, nodeid, remid);
5564        if (lkb)
5565                return lkb;
5566        lkb = search_remid_list(&r->res_waitqueue, nodeid, remid);
5567        if (lkb)
5568                return lkb;
5569        return NULL;
5570}
5571
5572/* needs at least dlm_rcom + rcom_lock */
5573static int receive_rcom_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
5574                                  struct dlm_rsb *r, struct dlm_rcom *rc)
5575{
5576        struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf;
5577
5578        lkb->lkb_nodeid = rc->rc_header.h_nodeid;
5579        lkb->lkb_ownpid = le32_to_cpu(rl->rl_ownpid);
5580        lkb->lkb_remid = le32_to_cpu(rl->rl_lkid);
5581        lkb->lkb_exflags = le32_to_cpu(rl->rl_exflags);
5582        lkb->lkb_flags = le32_to_cpu(rl->rl_flags) & 0x0000FFFF;
5583        lkb->lkb_flags |= DLM_IFL_MSTCPY;
5584        lkb->lkb_lvbseq = le32_to_cpu(rl->rl_lvbseq);
5585        lkb->lkb_rqmode = rl->rl_rqmode;
5586        lkb->lkb_grmode = rl->rl_grmode;
5587        /* don't set lkb_status because add_lkb wants to itself */
5588
5589        lkb->lkb_bastfn = (rl->rl_asts & DLM_CB_BAST) ? &fake_bastfn : NULL;
5590        lkb->lkb_astfn = (rl->rl_asts & DLM_CB_CAST) ? &fake_astfn : NULL;
5591
5592        if (lkb->lkb_exflags & DLM_LKF_VALBLK) {
5593                int lvblen = rc->rc_header.h_length - sizeof(struct dlm_rcom) -
5594                         sizeof(struct rcom_lock);
5595                if (lvblen > ls->ls_lvblen)
5596                        return -EINVAL;
5597                lkb->lkb_lvbptr = dlm_allocate_lvb(ls);
5598                if (!lkb->lkb_lvbptr)
5599                        return -ENOMEM;
5600                memcpy(lkb->lkb_lvbptr, rl->rl_lvb, lvblen);
5601        }
5602
5603        /* Conversions between PR and CW (middle modes) need special handling.
5604           The real granted mode of these converting locks cannot be determined
5605           until all locks have been rebuilt on the rsb (recover_conversion) */
5606
5607        if (rl->rl_wait_type == cpu_to_le16(DLM_MSG_CONVERT) &&
5608            middle_conversion(lkb)) {
5609                rl->rl_status = DLM_LKSTS_CONVERT;
5610                lkb->lkb_grmode = DLM_LOCK_IV;
5611                rsb_set_flag(r, RSB_RECOVER_CONVERT);
5612        }
5613
5614        return 0;
5615}
5616
5617/* This lkb may have been recovered in a previous aborted recovery so we need
5618   to check if the rsb already has an lkb with the given remote nodeid/lkid.
5619   If so we just send back a standard reply.  If not, we create a new lkb with
5620   the given values and send back our lkid.  We send back our lkid by sending
5621   back the rcom_lock struct we got but with the remid field filled in. */
5622
5623/* needs at least dlm_rcom + rcom_lock */
5624int dlm_recover_master_copy(struct dlm_ls