问题排查-线上CPU300%排查过程(dubbo引起的)

top

 ps -mp 39954 -o THREAD,tid,time|sort -r|head -n 30

printf "%x
" 40009
9c49

 jstack 39954|grep 9c49 -A 15

 com.alibaba.dubbo.registry.support.AbstractRegistry.doSaveProperties(AbstractRegistry.java:152)

    private final ExecutorService registryCacheExecutor = Executors.newFixedThreadPool(1, new NamedThreadFactory("DubboSaveRegistryCache", true));

   public void doSaveProperties(long version) {
        if (version < lastCacheChanged.get()) {
            return;
        }
        if (file == null) {
            return;
        }
        // Save
        try {
            File lockfile = new File(file.getAbsolutePath() + ".lock");
            //创建问题出现问题 抛出异常
            if (!lockfile.exists()) {
                lockfile.createNewFile();
            }
            //其他代码省略
        } catch (Throwable e) {
            if (version < lastCacheChanged.get()) {
                return;
            } else {
                //异常处理 继续进入该方法 形成死循环
                registryCacheExecutor.execute(new SaveProperties(lastCacheChanged.incrementAndGet()));
            }
            logger.warn("Failed to save registry store file, cause: " + e.getMessage(), e);
        }
    }

   private class SaveProperties implements Runnable {
        private long version;

        private SaveProperties(long version) {
            this.version = version;
        }

        @Override
        public void run() {
            doSaveProperties(version);
        }
    }