应用“假死”：CPU 2%，请求全挂，jstack 一眼看出两个线程在互相“谦让”

$ top -Hp 4187 -n 1 | head -12
  PID USER   PR  NI    VIRT    RES   SHR S  %CPU %MEM     TIME+ COMMAND
 4187 app    20   0 6231204   1.9g 25412 S   2.0 12.1  18:04.31 java
 4203 app    20   0 6231204   1.9g 25412 S   1.7  12.1   0:31.02 VM Periodic Tas
 4211 app    20   0 6231204   1.9g 25412 S   0.3  12.1   0:04.87 C2 CompilerThre
   ...（没有任何业务线程在吃 CPU）

$ jstack 4187 > /data/dumps/jstack-1.txt
$ grep 'java.lang.Thread.State' /data/dumps/jstack-1.txt | sort | uniq -c | sort -rn
     96    java.lang.Thread.State: BLOCKED (on object monitor)
     41    java.lang.Thread.State: WAITING (parking)
     12    java.lang.Thread.State: TIMED_WAITING (parking)
      9    java.lang.Thread.State: RUNNABLE

$ jstack 4187 | grep -A 40 'Found one Java-level deadlock'

Found one Java-level deadlock:
=============================
"http-nio-8080-exec-12":
  waiting to lock monitor 0x00007f3c1c004e00 (object 0x00000006c1a2b340, a java.lang.String),
  which is held by "http-nio-8080-exec-31"
"http-nio-8080-exec-31":
  waiting to lock monitor 0x00007f3c1c007750 (object 0x00000006c1a2b380, a java.lang.String),
  which is held by "http-nio-8080-exec-12"

Java stack information for the threads listed above:
===================================================
"http-nio-8080-exec-12":
	at com.demo.trade.service.StockService.deduct(StockService.java:52)
	- waiting to lock <0x00000006c1a2b340> (a java.lang.String)
	- locked <0x00000006c1a2b380> (a java.lang.String)
	at com.demo.trade.service.OrderService.create(OrderService.java:88)
	at com.demo.trade.web.OrderController.submit(OrderController.java:34)
"http-nio-8080-exec-31":
	at com.demo.trade.service.OrderService.cancelAndRestock(OrderService.java:141)
	- waiting to lock <0x00000006c1a2b380> (a java.lang.String)
	- locked <0x00000006c1a2b340> (a java.lang.String)
	at com.demo.trade.web.OrderController.cancel(OrderController.java:61)

Found 1 deadlock.

public class DeadlockDemo {
    private static final Object ORDER_LOCK = new Object();
    private static final Object STOCK_LOCK = new Object();

    public static void main(String[] args) {
        Thread create = new Thread(() -> {
            synchronized (STOCK_LOCK) {            // 下单：先库存
                sleep(100);                        // 放大竞争窗口，让另一个线程有机会拿到第二把锁
                synchronized (ORDER_LOCK) {        // 再订单
                    System.out.println("create done");
                }
            }
        }, "create-order");

        Thread cancel = new Thread(() -> {
            synchronized (ORDER_LOCK) {            // 取消：先订单
                sleep(100);
                synchronized (STOCK_LOCK) {        // 再库存
                    System.out.println("cancel done");
                }
            }
        }, "cancel-order");

        create.start();
        cancel.start();
    }

    private static void sleep(long ms) {
        try { Thread.sleep(ms); } catch (InterruptedException e) { Thread.currentThread().interrupt(); }
    }
}

$ javac DeadlockDemo.java && java DeadlockDemo &
$ jps -l
15320 DeadlockDemo
$ jstack 15320
...
Found one Java-level deadlock:
=============================
"cancel-order":
  waiting to lock monitor 0x000001d7e9b2d2d8 (object 0x00000000d5f7a3c0, a java.lang.Object),
  which is held by "create-order"
"create-order":
  waiting to lock monitor 0x000001d7e9b2f4a8 (object 0x00000000d5f7a3d0, a java.lang.Object),
  which is held by "cancel-order"

Java stack information for the threads listed above:
===================================================
"cancel-order":
	at DeadlockDemo.lambda$main$1(DeadlockDemo.java:21)
	- waiting to lock <0x00000000d5f7a3c0> (a java.lang.Object)
	- locked <0x00000000d5f7a3d0> (a java.lang.Object)
	at java.lang.Thread.run(Thread.java:750)
"create-order":
	at DeadlockDemo.lambda$main$0(DeadlockDemo.java:12)
	- waiting to lock <0x00000000d5f7a3d0> (a java.lang.Object)
	- locked <0x00000000d5f7a3c0> (a java.lang.Object)
	at java.lang.Thread.run(Thread.java:750)

Found 1 deadlock.

$ jstack -l 15320
"cancel-order" #13 prio=5 os_prio=0 tid=0x000001d7e9a8b000 nid=0x2f0c waiting on condition [0x000000de3c6ff000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000000d6a3c2e8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	...
   Locked ownable synchronizers:
	- <0x00000000d6a3c318> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)

Found one Java-level deadlock:
=============================
"cancel-order":
  waiting for ownable synchronizer 0x00000000d6a3c2e8, (a java.util.concurrent.locks.ReentrantLock$NonfairSync),
  which is held by "create-order"
...

// 一个简单的“死锁哨兵”：每分钟检测一次，发现就打印并告警
ScheduledExecutorService ses = Executors.newSingleThreadScheduledExecutor(r -> {
    Thread t = new Thread(r, "deadlock-watchdog"); t.setDaemon(true); return t; });
ThreadMXBean mx = ManagementFactory.getThreadMXBean();
ses.scheduleAtFixedRate(() -> {
    long[] ids = mx.findDeadlockedThreads();
    if (ids != null) {
        for (ThreadInfo ti : mx.getThreadInfo(ids, true, true)) {
            log.error("DEADLOCK detected: {}", ti);          // 含锁持有信息与栈
        }
        alarm("deadlock detected, ids=" + Arrays.toString(ids));
    }
}, 1, 1, TimeUnit.MINUTES);

// 下单：为了防止同一 SKU 超卖，先锁 SKU（库存），再锁订单号防重复提交
public void create(OrderDTO dto) {
    synchronized (("SKU:" + dto.getSkuId()).intern()) {          // 锁 1：库存
        synchronized (("ORDER:" + dto.getOrderNo()).intern()) {   // 锁 2：订单
            stockService.deduct(dto.getSkuId(), dto.getQty());
            orderMapper.insert(dto);
        }
    }
}

// 取消：为了防止并发取消，先锁订单，再回补库存要锁 SKU
public void cancelAndRestock(String orderNo) {
    synchronized (("ORDER:" + orderNo).intern()) {                // 锁 1：订单
        Order o = orderMapper.selectByNo(orderNo);
        synchronized (("SKU:" + o.getSkuId()).intern()) {         // 锁 2：库存
            stockService.restock(o.getSkuId(), o.getQty());
            orderMapper.markCancelled(orderNo);
        }
    }
}

// 需要同时锁 A、B 时，先排序再加锁，保证所有线程顺序一致
static void lockBoth(String a, String b, Runnable action) {
    String first = a.compareTo(b) <= 0 ? a : b;
    String second = first == a ? b : a;
    synchronized (LOCKS.computeIfAbsent(first, k -> new Object())) {          // LOCKS 是 ConcurrentHashMap<String, Object>，避免 intern
        synchronized (LOCKS.computeIfAbsent(second, k -> new Object())) {
            action.run();
        }
    }
}

boolean tryBoth(ReentrantLock l1, ReentrantLock l2, long timeoutMs, Runnable action) throws InterruptedException {
    if (!l1.tryLock(timeoutMs, TimeUnit.MILLISECONDS)) return false;    // 拿不到第一把：直接失败
    try {
        if (!l2.tryLock(timeoutMs, TimeUnit.MILLISECONDS)) return false; // 拿不到第二把：放弃，finally 会释放第一把
        try {
            action.run();
            return true;
        } finally {
            l2.unlock();
        }
    } finally {
        l1.unlock();
    }
}
// 调用方：返回 false 时，对用户提示“系统繁忙，请重试”，或加随机退避后重试 1~2 次（避免活锁）

-- 扣库存：一条语句完成检查 + 扣减，行锁由数据库持有，事务结束即释放
UPDATE t_stock SET available = available - #{qty}
 WHERE sku_id = #{skuId} AND available >= #{qty};
-- affected rows = 0 表示库存不足

-- 取消订单：状态机 CAS，重复取消只会有一次生效
UPDATE t_order SET status = 'CANCELLED'
 WHERE order_no = #{orderNo} AND status = 'CREATED';