Flock master startup.

masters/master-common-rules.mk

 # -*- makefile -*-
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 # This should be included by a makefile which lives in a buildmaster/buildslave
 # directory (next to the buildbot.tac file). That including makefile *must*
 # define MASTERPATH.
 
 # The 'start' and 'stop' targets start and stop the buildbot master.
 # The 'reconfig' target will tell a buildmaster to reload its config file.
 
 # Note that a relative PYTHONPATH entry is relative to the current directory.
 
 # Confirm that MASTERPATH has been defined.
 ifeq ($(MASTERPATH),)
   $(error MASTERPATH not defined.)
 endif
 
 # Use the puppet-managed infra-python CIPD deployment (which all masters have).
 INFRA_RUNPY = /opt/infra-python/run.py
 
 # Get the current host's short hostname.  We may use this in Makefiles that
 # include this file.
 SHORT_HOSTNAME := $(shell hostname -s)
 CURRENT_DIR = $(shell pwd)
 
+# Where we expect flock to live.
+FLOCK = /usr/bin/flock
+
+# Per-master lockfile.
+LOCKFILE = master_start.lock
+
 printstep:
 ifndef NO_REVISION_AUDIT
         @echo "**  `python -c 'import datetime; print datetime.datetime.utcnow().isoformat() + "Z"'`    make $(MAKECMDGOALS)" >> actions.log
         @pstree --show-parents $$$$ --ascii --arguments --show-pids >> actions.log
 endif
 
 notify:
         @if (hostname -f | grep -q '^master.*\.chromium\.org'); then \
                 /bin/echo ; \
                 /bin/echo -e "\033[1;31m***"; \
@@ skipping 18 matching lines @@
         $(GCLIENT) diff >> actions.log || true
         @($(INFRA_RUNPY) infra.tools.send_monitoring_event \
                    --service-event-revinfo=$(CURRENT_DIR)/revinfo.log \
                    --service-event-type=START \
                    --event-mon-run-type=prod \
                    --event-mon-service-name \
                    buildbot/master/$(MASTERPATH) \
    || echo 'Running send_monitoring_event failed, skipping sending events.' \
   ) 2>&1 | tee -a actions.log
 endif
-        # There is a race condition between startup and when twistd.pid is written.
-        # Anyone issuing a second `make start` before the twistd.pid is written will
-        # spawn two masters. This is hopefully unlikely, but a proper solution would
-        # be to use flock (not available on OSX). The critical section contains BOTH
-        # the twistd start and the wait for the twistd.pid.
-        @echo 'Now running Buildbot master.'
-        PYTHONPATH=$(PYTHONPATH) python $(SCRIPTS_DIR)/common/twistd --no_save -y buildbot.tac
-        @echo 'Waiting for creation of twistd.pid...'
-        while `test ! -f twistd.pid`; do sleep 1; done;
+ifneq ($(wildcard $(FLOCK)),)
+        PYTHONPATH=$(PYTHONPATH) SCRIPTS_DIR=$(SCRIPTS_DIR) $(FLOCK) -n $(LOCKFILE) $(TOPLEVEL_DIR)/masters/start_master.sh || ( \
+        echo "Failure to start master. Check to see if a master is running and" \
+             "holding the lock on $(LOCKFILE)."; exit 1)
+else
+        PYTHONPATH=$(PYTHONPATH) SCRIPTS_DIR=$(SCRIPTS_DIR) $(TOPLEVEL_DIR)/masters/start_master.sh
+endif
+
 
 ifeq ($(BUILDBOT_PATH),$(BUILDBOT8_PATH))
 start-prof: bootstrap
 else
 start-prof:
 endif
         TWISTD_PROFILE=1 PYTHONPATH=$(PYTHONPATH) python $(SCRIPTS_DIR)/common/twistd --no_save -y buildbot.tac
 
 stop: notify printstep
 ifndef NO_REVISION_AUDIT
@@ skipping 46 matching lines @@
         PYTHONPATH=$(PYTHONPATH) python buildbot upgrade-master .
 
 # This target is only known to be useful on 0.8.x masters.
 bootstrap: printstep
         @[ -e '.dbconfig' ] || [ -e 'state.sqlite' ] || \
         PYTHONPATH=$(PYTHONPATH) python $(SCRIPTS_DIR)/tools/state_create.py \
         --restore --db='state.sqlite' --txt '../state-template.txt'
 
 setup:
         @echo export PYTHONPATH=$(PYTHONPATH)